2024-12-10 07:14:28,463 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-10 07:14:28,485 main DEBUG Took 0.019552 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-10 07:14:28,486 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-10 07:14:28,487 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-10 07:14:28,488 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-10 07:14:28,490 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 07:14:28,502 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-10 07:14:28,519 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:14:28,520 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 07:14:28,521 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:14:28,522 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 07:14:28,523 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:14:28,523 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 07:14:28,526 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:14:28,527 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 07:14:28,527 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:14:28,528 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 07:14:28,529 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:14:28,529 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 07:14:28,530 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:14:28,530 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 07:14:28,531 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:14:28,531 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 07:14:28,532 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:14:28,532 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 07:14:28,532 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:14:28,533 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 07:14:28,533 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:14:28,534 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 07:14:28,534 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:14:28,534 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 07:14:28,535 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:14:28,535 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-10 07:14:28,537 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 07:14:28,539 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-10 07:14:28,541 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-10 07:14:28,542 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-10 07:14:28,543 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-10 07:14:28,544 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-10 07:14:28,558 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-10 07:14:28,562 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-10 07:14:28,564 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-10 07:14:28,565 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-10 07:14:28,566 main DEBUG createAppenders(={Console}) 2024-12-10 07:14:28,567 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-10 07:14:28,567 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-10 07:14:28,568 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-10 07:14:28,568 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-10 07:14:28,569 main DEBUG OutputStream closed 2024-12-10 07:14:28,569 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-10 07:14:28,570 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-10 07:14:28,570 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-10 07:14:28,700 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-10 07:14:28,703 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-10 07:14:28,704 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-10 07:14:28,705 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-10 07:14:28,706 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-10 07:14:28,706 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-10 07:14:28,707 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-10 07:14:28,707 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-10 07:14:28,708 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-10 07:14:28,708 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-10 07:14:28,709 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-10 07:14:28,709 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-10 07:14:28,710 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-10 07:14:28,710 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-10 07:14:28,711 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-10 07:14:28,711 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-10 07:14:28,711 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-10 07:14:28,713 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-10 07:14:28,716 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10 07:14:28,717 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-10 07:14:28,717 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-10 07:14:28,718 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-10T07:14:29,099 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3 2024-12-10 07:14:29,103 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-10 07:14:29,103 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10T07:14:29,115 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-10T07:14:29,148 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-10T07:14:29,183 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/cluster_ac824e32-1cbb-51f2-a594-edd33703f97a, deleteOnExit=true 2024-12-10T07:14:29,183 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-10T07:14:29,187 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/test.cache.data in system properties and HBase conf 2024-12-10T07:14:29,189 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/hadoop.tmp.dir in system properties and HBase conf 2024-12-10T07:14:29,190 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/hadoop.log.dir in system properties and HBase conf 2024-12-10T07:14:29,191 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-10T07:14:29,191 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-10T07:14:29,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-10T07:14:29,353 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-10T07:14:29,511 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T07:14:29,516 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-10T07:14:29,517 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-10T07:14:29,518 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-10T07:14:29,518 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-10T07:14:29,519 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-10T07:14:29,520 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-10T07:14:29,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-10T07:14:29,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-10T07:14:29,522 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-10T07:14:29,522 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/nfs.dump.dir in system properties and HBase conf 2024-12-10T07:14:29,523 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/java.io.tmpdir in system properties and HBase conf 2024-12-10T07:14:29,523 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-10T07:14:29,524 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-10T07:14:29,524 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-10T07:14:30,774 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-10T07:14:30,865 INFO [Time-limited test {}] log.Log(170): Logging initialized @3277ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-10T07:14:30,951 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T07:14:31,043 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-10T07:14:31,094 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-10T07:14:31,095 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-10T07:14:31,098 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-10T07:14:31,122 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T07:14:31,133 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/hadoop.log.dir/,AVAILABLE} 2024-12-10T07:14:31,135 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-10T07:14:31,428 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bbf242f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/java.io.tmpdir/jetty-localhost-37413-hadoop-hdfs-3_4_1-tests_jar-_-any-9392293664318093539/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-10T07:14:31,441 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:37413} 2024-12-10T07:14:31,442 INFO [Time-limited test {}] server.Server(415): Started @3854ms 2024-12-10T07:14:32,298 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T07:14:32,308 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-10T07:14:32,310 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-10T07:14:32,310 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-10T07:14:32,311 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-10T07:14:32,312 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6aa6ac55{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/hadoop.log.dir/,AVAILABLE} 2024-12-10T07:14:32,312 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@58f8c8ee{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-10T07:14:32,429 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4d1d0948{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/java.io.tmpdir/jetty-localhost-45953-hadoop-hdfs-3_4_1-tests_jar-_-any-2046395589904889968/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T07:14:32,430 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@49b49063{HTTP/1.1, (http/1.1)}{localhost:45953} 2024-12-10T07:14:32,431 INFO [Time-limited test {}] server.Server(415): Started @4843ms 2024-12-10T07:14:32,498 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-10T07:14:33,549 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/cluster_ac824e32-1cbb-51f2-a594-edd33703f97a/data/data1/current/BP-1692125974-172.17.0.3-1733814870213/current, will proceed with Du for space computation calculation, 2024-12-10T07:14:33,549 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/cluster_ac824e32-1cbb-51f2-a594-edd33703f97a/data/data2/current/BP-1692125974-172.17.0.3-1733814870213/current, will proceed with Du for space computation calculation, 2024-12-10T07:14:33,614 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-10T07:14:33,681 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf1c431286a80e9a2 with lease ID 0x7e59a6320992ff14: Processing first storage report for DS-71c3e2fe-e06c-4f2c-995a-a2f36a2ad6fa from datanode DatanodeRegistration(127.0.0.1:40613, datanodeUuid=992f94be-0c8f-440e-ad2d-537f43afb4bf, infoPort=41227, infoSecurePort=0, ipcPort=45679, storageInfo=lv=-57;cid=testClusterID;nsid=1020141547;c=1733814870213) 2024-12-10T07:14:33,683 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf1c431286a80e9a2 with lease ID 0x7e59a6320992ff14: from storage DS-71c3e2fe-e06c-4f2c-995a-a2f36a2ad6fa node DatanodeRegistration(127.0.0.1:40613, datanodeUuid=992f94be-0c8f-440e-ad2d-537f43afb4bf, infoPort=41227, infoSecurePort=0, ipcPort=45679, storageInfo=lv=-57;cid=testClusterID;nsid=1020141547;c=1733814870213), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-10T07:14:33,684 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf1c431286a80e9a2 with lease ID 0x7e59a6320992ff14: Processing first storage report for DS-baf56ae0-e268-4f47-ab5a-1cdb40e5cfd6 from datanode DatanodeRegistration(127.0.0.1:40613, datanodeUuid=992f94be-0c8f-440e-ad2d-537f43afb4bf, infoPort=41227, infoSecurePort=0, ipcPort=45679, storageInfo=lv=-57;cid=testClusterID;nsid=1020141547;c=1733814870213) 2024-12-10T07:14:33,684 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf1c431286a80e9a2 with lease ID 0x7e59a6320992ff14: from storage DS-baf56ae0-e268-4f47-ab5a-1cdb40e5cfd6 node DatanodeRegistration(127.0.0.1:40613, datanodeUuid=992f94be-0c8f-440e-ad2d-537f43afb4bf, infoPort=41227, infoSecurePort=0, ipcPort=45679, storageInfo=lv=-57;cid=testClusterID;nsid=1020141547;c=1733814870213), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-10T07:14:33,712 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3 2024-12-10T07:14:33,829 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/cluster_ac824e32-1cbb-51f2-a594-edd33703f97a/zookeeper_0, clientPort=62458, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/cluster_ac824e32-1cbb-51f2-a594-edd33703f97a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/cluster_ac824e32-1cbb-51f2-a594-edd33703f97a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-10T07:14:33,855 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62458 2024-12-10T07:14:33,876 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T07:14:33,880 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T07:14:34,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741825_1001 (size=7) 2024-12-10T07:14:34,542 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8 with version=8 2024-12-10T07:14:34,542 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/hbase-staging 2024-12-10T07:14:34,691 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-10T07:14:34,998 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b1396b8d2d12:0 server-side Connection retries=45 2024-12-10T07:14:35,015 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T07:14:35,016 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T07:14:35,024 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T07:14:35,025 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T07:14:35,025 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T07:14:35,219 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-10T07:14:35,292 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-10T07:14:35,304 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-10T07:14:35,310 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T07:14:35,338 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 5277 (auto-detected) 2024-12-10T07:14:35,341 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-10T07:14:35,362 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36259 2024-12-10T07:14:35,387 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36259 connecting to ZooKeeper ensemble=127.0.0.1:62458 2024-12-10T07:14:35,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:362590x0, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T07:14:35,536 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36259-0x1000e93758a0000 connected 2024-12-10T07:14:35,633 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T07:14:35,635 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T07:14:35,650 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T07:14:35,655 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8, hbase.cluster.distributed=false 2024-12-10T07:14:35,687 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T07:14:35,702 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36259 2024-12-10T07:14:35,703 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36259 2024-12-10T07:14:35,704 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36259 2024-12-10T07:14:35,743 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36259 2024-12-10T07:14:35,745 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36259 2024-12-10T07:14:35,872 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b1396b8d2d12:0 server-side Connection retries=45 2024-12-10T07:14:35,874 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T07:14:35,874 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T07:14:35,875 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T07:14:35,875 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T07:14:35,875 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T07:14:35,878 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T07:14:35,881 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T07:14:35,881 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35871 2024-12-10T07:14:35,883 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35871 connecting to ZooKeeper ensemble=127.0.0.1:62458 2024-12-10T07:14:35,885 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T07:14:35,889 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T07:14:35,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:358710x0, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T07:14:35,916 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:358710x0, quorum=127.0.0.1:62458, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T07:14:35,916 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35871-0x1000e93758a0001 connected 2024-12-10T07:14:35,920 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T07:14:35,928 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T07:14:35,931 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35871-0x1000e93758a0001, quorum=127.0.0.1:62458, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T07:14:35,937 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35871-0x1000e93758a0001, quorum=127.0.0.1:62458, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T07:14:35,938 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35871 2024-12-10T07:14:35,938 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35871 2024-12-10T07:14:35,939 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35871 2024-12-10T07:14:35,945 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35871 2024-12-10T07:14:35,945 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35871 2024-12-10T07:14:35,963 DEBUG [M:0;b1396b8d2d12:36259 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b1396b8d2d12:36259 2024-12-10T07:14:35,964 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b1396b8d2d12,36259,1733814874775 2024-12-10T07:14:35,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35871-0x1000e93758a0001, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T07:14:35,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T07:14:35,980 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b1396b8d2d12,36259,1733814874775 2024-12-10T07:14:36,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35871-0x1000e93758a0001, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T07:14:36,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T07:14:36,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35871-0x1000e93758a0001, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T07:14:36,015 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-10T07:14:36,016 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b1396b8d2d12,36259,1733814874775 from backup master directory 2024-12-10T07:14:36,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b1396b8d2d12,36259,1733814874775 2024-12-10T07:14:36,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35871-0x1000e93758a0001, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T07:14:36,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T07:14:36,031 WARN [master/b1396b8d2d12:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T07:14:36,032 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b1396b8d2d12,36259,1733814874775 2024-12-10T07:14:36,034 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-10T07:14:36,036 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-10T07:14:36,097 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/hbase.id] with ID: ff9dec8b-7c7e-4ce8-ab85-50b35520afdc 2024-12-10T07:14:36,097 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/.tmp/hbase.id 2024-12-10T07:14:36,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741826_1002 (size=42) 2024-12-10T07:14:36,109 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/.tmp/hbase.id]:[hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/hbase.id] 2024-12-10T07:14:36,153 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T07:14:36,158 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-10T07:14:36,177 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-10T07:14:36,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T07:14:36,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35871-0x1000e93758a0001, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T07:14:36,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741827_1003 (size=196) 2024-12-10T07:14:36,220 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T07:14:36,223 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-10T07:14:36,238 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:36,245 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T07:14:36,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741828_1004 (size=1189) 2024-12-10T07:14:36,698 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData/data/master/store 2024-12-10T07:14:36,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741829_1005 (size=34) 2024-12-10T07:14:37,129 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-10T07:14:37,132 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T07:14:37,133 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T07:14:37,133 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T07:14:37,133 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T07:14:37,135 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T07:14:37,135 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T07:14:37,135 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T07:14:37,138 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733814877133Disabling compacts and flushes for region at 1733814877133Disabling writes for close at 1733814877135 (+2 ms)Writing region close event to WAL at 1733814877135Closed at 1733814877135 2024-12-10T07:14:37,140 WARN [master/b1396b8d2d12:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData/data/master/store/.initializing 2024-12-10T07:14:37,140 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData/WALs/b1396b8d2d12,36259,1733814874775 2024-12-10T07:14:37,150 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T07:14:37,167 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b1396b8d2d12%2C36259%2C1733814874775, suffix=, logDir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData/WALs/b1396b8d2d12,36259,1733814874775, archiveDir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData/oldWALs, maxLogs=10 2024-12-10T07:14:37,190 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData/WALs/b1396b8d2d12,36259,1733814874775/b1396b8d2d12%2C36259%2C1733814874775.1733814877172, exclude list is [], retry=0 2024-12-10T07:14:37,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40613,DS-71c3e2fe-e06c-4f2c-995a-a2f36a2ad6fa,DISK] 2024-12-10T07:14:37,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-10T07:14:37,253 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData/WALs/b1396b8d2d12,36259,1733814874775/b1396b8d2d12%2C36259%2C1733814874775.1733814877172 2024-12-10T07:14:37,257 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41227:41227)] 2024-12-10T07:14:37,258 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-10T07:14:37,259 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T07:14:37,262 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T07:14:37,264 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T07:14:37,302 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T07:14:37,332 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-10T07:14:37,337 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:14:37,340 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T07:14:37,341 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T07:14:37,344 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-10T07:14:37,345 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:14:37,346 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:14:37,346 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T07:14:37,349 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-10T07:14:37,349 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:14:37,350 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:14:37,350 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T07:14:37,354 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-10T07:14:37,354 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:14:37,355 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:14:37,356 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T07:14:37,360 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T07:14:37,362 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T07:14:37,368 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T07:14:37,369 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T07:14:37,374 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T07:14:37,379 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T07:14:37,385 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T07:14:37,387 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68569008, jitterRate=0.021757841110229492}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T07:14:37,394 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733814877277Initializing all the Stores at 1733814877280 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733814877280Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733814877281 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733814877281Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733814877281Cleaning up temporary data from old regions at 1733814877369 (+88 ms)Region opened successfully at 1733814877393 (+24 ms) 2024-12-10T07:14:37,395 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-10T07:14:37,440 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a8efb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b1396b8d2d12/172.17.0.3:0 2024-12-10T07:14:37,480 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-10T07:14:37,495 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-10T07:14:37,495 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-10T07:14:37,499 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-10T07:14:37,501 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-12-10T07:14:37,508 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-10T07:14:37,508 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-10T07:14:37,541 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-10T07:14:37,552 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-10T07:14:37,620 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-10T07:14:37,622 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-10T07:14:37,624 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-10T07:14:37,636 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-10T07:14:37,639 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-10T07:14:37,642 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-10T07:14:37,651 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-10T07:14:37,653 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-10T07:14:37,662 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-10T07:14:37,680 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-10T07:14:37,688 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-10T07:14:37,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T07:14:37,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35871-0x1000e93758a0001, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T07:14:37,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T07:14:37,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35871-0x1000e93758a0001, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T07:14:37,707 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b1396b8d2d12,36259,1733814874775, sessionid=0x1000e93758a0000, setting cluster-up flag (Was=false) 2024-12-10T07:14:37,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35871-0x1000e93758a0001, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T07:14:37,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T07:14:37,767 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-10T07:14:37,769 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b1396b8d2d12,36259,1733814874775 2024-12-10T07:14:37,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T07:14:37,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35871-0x1000e93758a0001, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T07:14:37,826 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-10T07:14:37,828 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b1396b8d2d12,36259,1733814874775 2024-12-10T07:14:37,834 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-10T07:14:37,854 INFO [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer(746): ClusterId : ff9dec8b-7c7e-4ce8-ab85-50b35520afdc 2024-12-10T07:14:37,856 DEBUG [RS:0;b1396b8d2d12:35871 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T07:14:37,879 DEBUG [RS:0;b1396b8d2d12:35871 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T07:14:37,879 DEBUG [RS:0;b1396b8d2d12:35871 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T07:14:37,897 DEBUG [RS:0;b1396b8d2d12:35871 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T07:14:37,898 DEBUG [RS:0;b1396b8d2d12:35871 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ebef31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b1396b8d2d12/172.17.0.3:0 2024-12-10T07:14:37,917 DEBUG [RS:0;b1396b8d2d12:35871 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b1396b8d2d12:35871 2024-12-10T07:14:37,922 INFO [RS:0;b1396b8d2d12:35871 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T07:14:37,922 INFO [RS:0;b1396b8d2d12:35871 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T07:14:37,922 DEBUG [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T07:14:37,928 INFO [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer(2659): reportForDuty to master=b1396b8d2d12,36259,1733814874775 with port=35871, startcode=1733814875835 2024-12-10T07:14:37,942 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-10T07:14:37,943 DEBUG [RS:0;b1396b8d2d12:35871 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T07:14:37,957 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-10T07:14:37,965 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-10T07:14:37,972 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b1396b8d2d12,36259,1733814874775 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-10T07:14:37,995 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b1396b8d2d12:0, corePoolSize=5, maxPoolSize=5 2024-12-10T07:14:37,996 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b1396b8d2d12:0, corePoolSize=5, maxPoolSize=5 2024-12-10T07:14:37,996 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b1396b8d2d12:0, corePoolSize=5, maxPoolSize=5 2024-12-10T07:14:37,996 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b1396b8d2d12:0, corePoolSize=5, maxPoolSize=5 2024-12-10T07:14:37,997 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b1396b8d2d12:0, corePoolSize=10, maxPoolSize=10 2024-12-10T07:14:37,997 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b1396b8d2d12:0, corePoolSize=1, maxPoolSize=1 2024-12-10T07:14:37,997 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b1396b8d2d12:0, corePoolSize=2, maxPoolSize=2 2024-12-10T07:14:37,998 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b1396b8d2d12:0, corePoolSize=1, maxPoolSize=1 2024-12-10T07:14:38,006 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733814908005 2024-12-10T07:14:38,006 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48473, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T07:14:38,008 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-10T07:14:38,009 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-10T07:14:38,013 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T07:14:38,013 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36259 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:38,014 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-10T07:14:38,021 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-10T07:14:38,022 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-10T07:14:38,022 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:14:38,022 INFO [PEWorker-2 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-10T07:14:38,025 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-10T07:14:38,025 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-10T07:14:38,026 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T07:14:38,033 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-10T07:14:38,034 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-10T07:14:38,035 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-10T07:14:38,046 DEBUG [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-10T07:14:38,046 WARN [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-10T07:14:38,047 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-10T07:14:38,047 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-10T07:14:38,049 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b1396b8d2d12:0:becomeActiveMaster-HFileCleaner.large.0-1733814878049,5,FailOnTimeoutGroup] 2024-12-10T07:14:38,050 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b1396b8d2d12:0:becomeActiveMaster-HFileCleaner.small.0-1733814878050,5,FailOnTimeoutGroup] 2024-12-10T07:14:38,050 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T07:14:38,050 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-10T07:14:38,051 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-10T07:14:38,052 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-10T07:14:38,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741831_1007 (size=1321) 2024-12-10T07:14:38,148 INFO [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer(2659): reportForDuty to master=b1396b8d2d12,36259,1733814874775 with port=35871, startcode=1733814875835 2024-12-10T07:14:38,150 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36259 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:38,153 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36259 {}] master.ServerManager(517): Registering regionserver=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:38,160 DEBUG [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8 2024-12-10T07:14:38,160 DEBUG [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38619 2024-12-10T07:14:38,160 DEBUG [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T07:14:38,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T07:14:38,174 DEBUG [RS:0;b1396b8d2d12:35871 {}] zookeeper.ZKUtil(111): regionserver:35871-0x1000e93758a0001, quorum=127.0.0.1:62458, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:38,174 WARN [RS:0;b1396b8d2d12:35871 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T07:14:38,175 INFO [RS:0;b1396b8d2d12:35871 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T07:14:38,175 DEBUG [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/WALs/b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:38,183 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b1396b8d2d12,35871,1733814875835] 2024-12-10T07:14:38,201 INFO [RS:0;b1396b8d2d12:35871 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T07:14:38,215 INFO [RS:0;b1396b8d2d12:35871 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T07:14:38,221 INFO [RS:0;b1396b8d2d12:35871 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T07:14:38,221 INFO [RS:0;b1396b8d2d12:35871 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T07:14:38,223 INFO [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T07:14:38,229 INFO [RS:0;b1396b8d2d12:35871 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T07:14:38,231 INFO [RS:0;b1396b8d2d12:35871 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T07:14:38,231 DEBUG [RS:0;b1396b8d2d12:35871 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b1396b8d2d12:0, corePoolSize=1, maxPoolSize=1 2024-12-10T07:14:38,231 DEBUG [RS:0;b1396b8d2d12:35871 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b1396b8d2d12:0, corePoolSize=1, maxPoolSize=1 2024-12-10T07:14:38,231 DEBUG [RS:0;b1396b8d2d12:35871 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b1396b8d2d12:0, corePoolSize=1, maxPoolSize=1 2024-12-10T07:14:38,231 DEBUG [RS:0;b1396b8d2d12:35871 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b1396b8d2d12:0, corePoolSize=1, maxPoolSize=1 2024-12-10T07:14:38,232 DEBUG [RS:0;b1396b8d2d12:35871 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b1396b8d2d12:0, corePoolSize=1, maxPoolSize=1 2024-12-10T07:14:38,232 DEBUG [RS:0;b1396b8d2d12:35871 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b1396b8d2d12:0, corePoolSize=2, maxPoolSize=2 2024-12-10T07:14:38,232 DEBUG [RS:0;b1396b8d2d12:35871 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b1396b8d2d12:0, corePoolSize=1, maxPoolSize=1 2024-12-10T07:14:38,232 DEBUG [RS:0;b1396b8d2d12:35871 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b1396b8d2d12:0, corePoolSize=1, maxPoolSize=1 2024-12-10T07:14:38,232 DEBUG [RS:0;b1396b8d2d12:35871 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b1396b8d2d12:0, corePoolSize=1, maxPoolSize=1 2024-12-10T07:14:38,233 DEBUG [RS:0;b1396b8d2d12:35871 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b1396b8d2d12:0, corePoolSize=1, maxPoolSize=1 2024-12-10T07:14:38,233 DEBUG [RS:0;b1396b8d2d12:35871 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b1396b8d2d12:0, corePoolSize=1, maxPoolSize=1 2024-12-10T07:14:38,233 DEBUG [RS:0;b1396b8d2d12:35871 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b1396b8d2d12:0, corePoolSize=1, maxPoolSize=1 2024-12-10T07:14:38,233 DEBUG [RS:0;b1396b8d2d12:35871 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b1396b8d2d12:0, corePoolSize=3, maxPoolSize=3 2024-12-10T07:14:38,233 DEBUG [RS:0;b1396b8d2d12:35871 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0, corePoolSize=3, maxPoolSize=3 2024-12-10T07:14:38,234 INFO [RS:0;b1396b8d2d12:35871 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T07:14:38,235 INFO [RS:0;b1396b8d2d12:35871 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T07:14:38,235 INFO [RS:0;b1396b8d2d12:35871 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T07:14:38,235 INFO [RS:0;b1396b8d2d12:35871 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T07:14:38,235 INFO [RS:0;b1396b8d2d12:35871 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T07:14:38,235 INFO [RS:0;b1396b8d2d12:35871 {}] hbase.ChoreService(168): Chore ScheduledChore name=b1396b8d2d12,35871,1733814875835-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T07:14:38,259 INFO [RS:0;b1396b8d2d12:35871 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T07:14:38,262 INFO [RS:0;b1396b8d2d12:35871 {}] hbase.ChoreService(168): Chore ScheduledChore name=b1396b8d2d12,35871,1733814875835-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T07:14:38,262 INFO [RS:0;b1396b8d2d12:35871 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T07:14:38,263 INFO [RS:0;b1396b8d2d12:35871 {}] regionserver.Replication(171): b1396b8d2d12,35871,1733814875835 started 2024-12-10T07:14:38,288 INFO [RS:0;b1396b8d2d12:35871 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T07:14:38,289 INFO [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer(1482): Serving as b1396b8d2d12,35871,1733814875835, RpcServer on b1396b8d2d12/172.17.0.3:35871, sessionid=0x1000e93758a0001 2024-12-10T07:14:38,289 DEBUG [RS:0;b1396b8d2d12:35871 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T07:14:38,290 DEBUG [RS:0;b1396b8d2d12:35871 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:38,290 DEBUG [RS:0;b1396b8d2d12:35871 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b1396b8d2d12,35871,1733814875835' 2024-12-10T07:14:38,290 DEBUG [RS:0;b1396b8d2d12:35871 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T07:14:38,294 DEBUG [RS:0;b1396b8d2d12:35871 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T07:14:38,295 DEBUG [RS:0;b1396b8d2d12:35871 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T07:14:38,295 DEBUG [RS:0;b1396b8d2d12:35871 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T07:14:38,295 DEBUG [RS:0;b1396b8d2d12:35871 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:38,296 DEBUG [RS:0;b1396b8d2d12:35871 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b1396b8d2d12,35871,1733814875835' 2024-12-10T07:14:38,296 DEBUG [RS:0;b1396b8d2d12:35871 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T07:14:38,296 DEBUG [RS:0;b1396b8d2d12:35871 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T07:14:38,297 DEBUG [RS:0;b1396b8d2d12:35871 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T07:14:38,297 INFO [RS:0;b1396b8d2d12:35871 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T07:14:38,297 INFO [RS:0;b1396b8d2d12:35871 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T07:14:38,402 INFO [RS:0;b1396b8d2d12:35871 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T07:14:38,405 INFO [RS:0;b1396b8d2d12:35871 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b1396b8d2d12%2C35871%2C1733814875835, suffix=, logDir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/WALs/b1396b8d2d12,35871,1733814875835, archiveDir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/oldWALs, maxLogs=32 2024-12-10T07:14:38,424 DEBUG [RS:0;b1396b8d2d12:35871 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/WALs/b1396b8d2d12,35871,1733814875835/b1396b8d2d12%2C35871%2C1733814875835.1733814878408, exclude list is [], retry=0 2024-12-10T07:14:38,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40613,DS-71c3e2fe-e06c-4f2c-995a-a2f36a2ad6fa,DISK] 2024-12-10T07:14:38,433 INFO [RS:0;b1396b8d2d12:35871 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/WALs/b1396b8d2d12,35871,1733814875835/b1396b8d2d12%2C35871%2C1733814875835.1733814878408 2024-12-10T07:14:38,436 DEBUG [RS:0;b1396b8d2d12:35871 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41227:41227)] 2024-12-10T07:14:38,459 INFO [PEWorker-2 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-10T07:14:38,460 INFO [PEWorker-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8 2024-12-10T07:14:38,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741833_1009 (size=32) 2024-12-10T07:14:38,472 DEBUG [PEWorker-2 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T07:14:38,476 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-10T07:14:38,479 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T07:14:38,479 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:14:38,480 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T07:14:38,480 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-10T07:14:38,483 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T07:14:38,483 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:14:38,484 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T07:14:38,484 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-10T07:14:38,487 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T07:14:38,488 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:14:38,489 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T07:14:38,489 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-10T07:14:38,492 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T07:14:38,493 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:14:38,494 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T07:14:38,495 DEBUG [PEWorker-2 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T07:14:38,497 DEBUG [PEWorker-2 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/1588230740 2024-12-10T07:14:38,498 DEBUG [PEWorker-2 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/1588230740 2024-12-10T07:14:38,501 DEBUG [PEWorker-2 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T07:14:38,501 DEBUG [PEWorker-2 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T07:14:38,503 DEBUG [PEWorker-2 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T07:14:38,506 DEBUG [PEWorker-2 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T07:14:38,511 DEBUG [PEWorker-2 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T07:14:38,512 INFO [PEWorker-2 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68678623, jitterRate=0.023391231894493103}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T07:14:38,515 DEBUG [PEWorker-2 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733814878473Initializing all the Stores at 1733814878474 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733814878475 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733814878475Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733814878475Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733814878475Cleaning up temporary data from old regions at 1733814878502 (+27 ms)Region opened successfully at 1733814878514 (+12 ms) 2024-12-10T07:14:38,515 DEBUG [PEWorker-2 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T07:14:38,515 INFO [PEWorker-2 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T07:14:38,515 DEBUG [PEWorker-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T07:14:38,515 DEBUG [PEWorker-2 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T07:14:38,515 DEBUG [PEWorker-2 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T07:14:38,517 INFO [PEWorker-2 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T07:14:38,517 DEBUG [PEWorker-2 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733814878515Disabling compacts and flushes for region at 1733814878515Disabling writes for close at 1733814878515Writing region close event to WAL at 1733814878516 (+1 ms)Closed at 1733814878517 (+1 ms) 2024-12-10T07:14:38,521 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T07:14:38,521 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-10T07:14:38,528 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-10T07:14:38,544 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T07:14:38,547 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-10T07:14:38,700 DEBUG [b1396b8d2d12:36259 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-10T07:14:38,714 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:38,720 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b1396b8d2d12,35871,1733814875835, state=OPENING 2024-12-10T07:14:38,767 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-10T07:14:38,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T07:14:38,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35871-0x1000e93758a0001, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T07:14:38,779 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T07:14:38,779 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T07:14:38,781 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T07:14:38,783 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b1396b8d2d12,35871,1733814875835}] 2024-12-10T07:14:38,958 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T07:14:38,962 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34895, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T07:14:38,974 INFO [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-10T07:14:38,974 INFO [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T07:14:38,975 INFO [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-10T07:14:38,980 INFO [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b1396b8d2d12%2C35871%2C1733814875835.meta, suffix=.meta, logDir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/WALs/b1396b8d2d12,35871,1733814875835, archiveDir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/oldWALs, maxLogs=32 2024-12-10T07:14:39,003 DEBUG [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/WALs/b1396b8d2d12,35871,1733814875835/b1396b8d2d12%2C35871%2C1733814875835.meta.1733814878982.meta, exclude list is [], retry=0 2024-12-10T07:14:39,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40613,DS-71c3e2fe-e06c-4f2c-995a-a2f36a2ad6fa,DISK] 2024-12-10T07:14:39,011 INFO [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/WALs/b1396b8d2d12,35871,1733814875835/b1396b8d2d12%2C35871%2C1733814875835.meta.1733814878982.meta 2024-12-10T07:14:39,011 DEBUG [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41227:41227)] 2024-12-10T07:14:39,011 DEBUG [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-10T07:14:39,013 DEBUG [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-10T07:14:39,016 DEBUG [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-10T07:14:39,021 INFO [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-10T07:14:39,026 DEBUG [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-10T07:14:39,026 DEBUG [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T07:14:39,027 DEBUG [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-10T07:14:39,027 DEBUG [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-10T07:14:39,030 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-10T07:14:39,032 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T07:14:39,032 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:14:39,034 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T07:14:39,034 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-10T07:14:39,036 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T07:14:39,036 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:14:39,038 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T07:14:39,038 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-10T07:14:39,040 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T07:14:39,040 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:14:39,042 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T07:14:39,042 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-10T07:14:39,044 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T07:14:39,044 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:14:39,045 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T07:14:39,045 DEBUG [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T07:14:39,047 DEBUG [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/1588230740 2024-12-10T07:14:39,051 DEBUG [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/1588230740 2024-12-10T07:14:39,054 DEBUG [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T07:14:39,054 DEBUG [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T07:14:39,057 DEBUG [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T07:14:39,061 DEBUG [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T07:14:39,065 INFO [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63894982, jitterRate=-0.04789057374000549}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T07:14:39,066 DEBUG [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-10T07:14:39,068 DEBUG [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733814879027Writing region info on filesystem at 1733814879028 (+1 ms)Initializing all the Stores at 1733814879029 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733814879030 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733814879030Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733814879030Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733814879030Cleaning up temporary data from old regions at 1733814879054 (+24 ms)Running coprocessor post-open hooks at 1733814879066 (+12 ms)Region opened successfully at 1733814879068 (+2 ms) 2024-12-10T07:14:39,075 INFO [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733814878950 2024-12-10T07:14:39,087 DEBUG [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-10T07:14:39,088 INFO [RS_OPEN_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-10T07:14:39,090 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:39,093 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b1396b8d2d12,35871,1733814875835, state=OPEN 2024-12-10T07:14:39,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T07:14:39,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35871-0x1000e93758a0001, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T07:14:39,235 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T07:14:39,235 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T07:14:39,236 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:39,246 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-10T07:14:39,246 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b1396b8d2d12,35871,1733814875835 in 453 msec 2024-12-10T07:14:39,254 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-10T07:14:39,254 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 719 msec 2024-12-10T07:14:39,255 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T07:14:39,256 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-10T07:14:39,274 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:14:39,275 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:14:39,293 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:14:39,295 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51153, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:14:39,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.4420 sec 2024-12-10T07:14:39,316 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733814879315, completionTime=-1 2024-12-10T07:14:39,318 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-10T07:14:39,318 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-10T07:14:39,343 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-10T07:14:39,343 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733814939343 2024-12-10T07:14:39,344 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733814999343 2024-12-10T07:14:39,344 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-12-10T07:14:39,346 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b1396b8d2d12,36259,1733814874775-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T07:14:39,346 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b1396b8d2d12,36259,1733814874775-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T07:14:39,346 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b1396b8d2d12,36259,1733814874775-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T07:14:39,348 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b1396b8d2d12:36259, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T07:14:39,348 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-10T07:14:39,349 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-10T07:14:39,354 DEBUG [master/b1396b8d2d12:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-10T07:14:39,373 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.341sec 2024-12-10T07:14:39,374 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-10T07:14:39,375 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-10T07:14:39,376 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-10T07:14:39,377 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-10T07:14:39,377 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-10T07:14:39,378 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b1396b8d2d12,36259,1733814874775-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T07:14:39,378 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b1396b8d2d12,36259,1733814874775-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-10T07:14:39,387 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-10T07:14:39,388 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-10T07:14:39,389 INFO [master/b1396b8d2d12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b1396b8d2d12,36259,1733814874775-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T07:14:39,463 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4799df63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:39,465 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-10T07:14:39,465 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-10T07:14:39,467 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:14:39,470 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:14:39,478 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:14:39,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:14:39,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:14:39,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@442c9f8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:39,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:14:39,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:14:39,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:14:39,487 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:14:39,490 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f66000e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:39,491 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:14:39,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:14:39,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:14:39,502 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:14:39,520 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=b1396b8d2d12,36259,1733814874775 2024-12-10T07:14:39,533 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=716, ProcessCount=11, AvailableMemoryMB=4078 2024-12-10T07:14:39,541 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T07:14:39,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is b1396b8d2d12,36259,1733814874775 2024-12-10T07:14:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@757aa243 2024-12-10T07:14:39,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T07:14:39,552 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T07:14:39,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T07:14:39,559 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T07:14:39,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T07:14:39,570 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T07:14:39,572 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-10T07:14:39,572 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:14:39,575 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T07:14:39,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T07:14:39,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741835_1011 (size=1061) 2024-12-10T07:14:39,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T07:14:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T07:14:39,989 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8 2024-12-10T07:14:39,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741836_1012 (size=53) 2024-12-10T07:14:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T07:14:40,401 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T07:14:40,401 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4204f23293292645e34a5d93616d8bf1, disabling compactions & flushes 2024-12-10T07:14:40,401 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:40,401 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:40,402 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. after waiting 0 ms 2024-12-10T07:14:40,402 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:40,402 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:40,402 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4204f23293292645e34a5d93616d8bf1: Waiting for close lock at 1733814880401Disabling compacts and flushes for region at 1733814880401Disabling writes for close at 1733814880402 (+1 ms)Writing region close event to WAL at 1733814880402Closed at 1733814880402 2024-12-10T07:14:40,409 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T07:14:40,416 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733814880409"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733814880409"}]},"ts":"1733814880409"} 2024-12-10T07:14:40,421 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T07:14:40,422 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T07:14:40,425 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733814880423"}]},"ts":"1733814880423"} 2024-12-10T07:14:40,430 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T07:14:40,432 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4204f23293292645e34a5d93616d8bf1, ASSIGN}] 2024-12-10T07:14:40,434 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4204f23293292645e34a5d93616d8bf1, ASSIGN 2024-12-10T07:14:40,437 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4204f23293292645e34a5d93616d8bf1, ASSIGN; state=OFFLINE, location=b1396b8d2d12,35871,1733814875835; forceNewPlan=false, retain=false 2024-12-10T07:14:40,588 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4204f23293292645e34a5d93616d8bf1, regionState=OPENING, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:40,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4204f23293292645e34a5d93616d8bf1, ASSIGN because future has completed 2024-12-10T07:14:40,594 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835}] 2024-12-10T07:14:40,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T07:14:40,760 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:40,760 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} 2024-12-10T07:14:40,761 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:40,761 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T07:14:40,762 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:40,762 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:40,765 INFO [StoreOpener-4204f23293292645e34a5d93616d8bf1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:40,769 INFO [StoreOpener-4204f23293292645e34a5d93616d8bf1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:14:40,769 INFO [StoreOpener-4204f23293292645e34a5d93616d8bf1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4204f23293292645e34a5d93616d8bf1 columnFamilyName A 2024-12-10T07:14:40,769 DEBUG [StoreOpener-4204f23293292645e34a5d93616d8bf1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:14:40,771 INFO [StoreOpener-4204f23293292645e34a5d93616d8bf1-1 {}] regionserver.HStore(327): Store=4204f23293292645e34a5d93616d8bf1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:14:40,771 INFO [StoreOpener-4204f23293292645e34a5d93616d8bf1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:40,773 INFO [StoreOpener-4204f23293292645e34a5d93616d8bf1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:14:40,773 INFO [StoreOpener-4204f23293292645e34a5d93616d8bf1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4204f23293292645e34a5d93616d8bf1 columnFamilyName B 2024-12-10T07:14:40,773 DEBUG [StoreOpener-4204f23293292645e34a5d93616d8bf1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:14:40,774 INFO [StoreOpener-4204f23293292645e34a5d93616d8bf1-1 {}] regionserver.HStore(327): Store=4204f23293292645e34a5d93616d8bf1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:14:40,775 INFO [StoreOpener-4204f23293292645e34a5d93616d8bf1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:40,777 INFO [StoreOpener-4204f23293292645e34a5d93616d8bf1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:14:40,777 INFO [StoreOpener-4204f23293292645e34a5d93616d8bf1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4204f23293292645e34a5d93616d8bf1 columnFamilyName C 2024-12-10T07:14:40,777 DEBUG [StoreOpener-4204f23293292645e34a5d93616d8bf1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:14:40,778 INFO [StoreOpener-4204f23293292645e34a5d93616d8bf1-1 {}] regionserver.HStore(327): Store=4204f23293292645e34a5d93616d8bf1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:14:40,779 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:40,779 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:40,780 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:40,781 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:40,783 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:40,783 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:40,784 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T07:14:40,786 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:40,790 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T07:14:40,791 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 4204f23293292645e34a5d93616d8bf1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69645210, jitterRate=0.03779450058937073}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T07:14:40,791 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:40,792 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 4204f23293292645e34a5d93616d8bf1: Running coprocessor pre-open hook at 1733814880762Writing region info on filesystem at 1733814880762Initializing all the Stores at 1733814880764 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733814880764Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733814880764Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733814880765 (+1 ms)Cleaning up temporary data from old regions at 1733814880783 (+18 ms)Running coprocessor post-open hooks at 1733814880791 (+8 ms)Region opened successfully at 1733814880792 (+1 ms) 2024-12-10T07:14:40,794 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., pid=6, masterSystemTime=1733814880749 2024-12-10T07:14:40,797 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:40,797 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:40,798 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4204f23293292645e34a5d93616d8bf1, regionState=OPEN, openSeqNum=2, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:40,802 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 because future has completed 2024-12-10T07:14:40,809 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-10T07:14:40,811 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 in 210 msec 2024-12-10T07:14:40,814 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-10T07:14:40,814 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4204f23293292645e34a5d93616d8bf1, ASSIGN in 378 msec 2024-12-10T07:14:40,816 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T07:14:40,816 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733814880816"}]},"ts":"1733814880816"} 2024-12-10T07:14:40,820 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T07:14:40,822 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T07:14:40,826 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2600 sec 2024-12-10T07:14:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T07:14:41,736 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:14:41,741 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39b7cf82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,741 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:14:41,741 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:14:41,743 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:14:41,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:14:41,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:14:41,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f534466, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:14:41,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:14:41,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:14:41,746 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:14:41,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@297ba807, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,748 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:14:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:14:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:14:41,752 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:14:41,754 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T07:14:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is b1396b8d2d12,36259,1733814874775 2024-12-10T07:14:41,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@52a4d08 2024-12-10T07:14:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T07:14:41,760 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T07:14:41,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c22fa37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,764 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:14:41,764 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:14:41,766 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:14:41,766 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:14:41,766 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:14:41,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55ae7626, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:14:41,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:14:41,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:14:41,769 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:14:41,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c5b0754, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,771 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d7b17c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,771 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:14:41,771 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:14:41,773 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:14:41,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:14:41,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:14:41,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e855a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:14:41,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:14:41,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:14:41,777 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:14:41,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a5a580f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6068bcec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,778 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:14:41,779 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:14:41,780 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:14:41,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:14:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:14:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27cf2c02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:14:41,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:14:41,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:14:41,784 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:14:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0d5e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,786 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11fbf093, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,786 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:14:41,786 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:14:41,787 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:14:41,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:14:41,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:14:41,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12d79f97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:14:41,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:14:41,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:14:41,791 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:14:41,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51273011, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64f1c2cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,793 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:14:41,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:14:41,794 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:14:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:14:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:14:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42ace511, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:14:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:14:41,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:14:41,797 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:14:41,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68bfa200, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,799 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eea3ba9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,799 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:14:41,799 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:14:41,800 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:14:41,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:14:41,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:14:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f759963, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:14:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:14:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:14:41,803 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:14:41,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28c8a4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d6a39a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,805 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:14:41,805 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:14:41,806 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:14:41,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:14:41,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:14:41,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56b0da03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:14:41,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:14:41,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:14:41,809 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:14:41,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc04dad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,811 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29b7950b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,811 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:14:41,811 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:14:41,812 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:14:41,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:14:41,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:14:41,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74b54236, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:14:41,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:14:41,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:14:41,815 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:14:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c1b1a46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b18ef51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,817 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:14:41,817 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:14:41,819 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:14:41,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:14:41,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:14:41,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d424882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:14:41,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:14:41,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:14:41,822 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:14:41,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@293d4ff4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:14:41,827 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:14:41,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:14:41,830 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:14:41,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:14:41,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:14:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:14:41,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:14:41,836 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:14:41,837 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:14:41,840 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:14:41,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:14:41,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:14:41,841 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:14:41,843 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:14:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:14:41,849 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:14:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:14:41,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:14:41,854 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:14:41,854 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:14:41,856 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:14:41,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:14:41,858 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:14:41,860 DEBUG [Thread-151 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:14:41,860 DEBUG [Thread-151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:14:41,862 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:14:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:14:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:14:41,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-10T07:14:41,866 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:14:41,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:14:41,867 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:14:41,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T07:14:41,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:14:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:14:41,872 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:14:41,873 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:14:41,875 DEBUG [Thread-143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:14:41,875 DEBUG [Thread-143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:14:41,876 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:14:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:14:41,878 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:14:41,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:14:41,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:14:41,882 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:14:41,882 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:14:41,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:14:41,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:14:41,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:14:41,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:14:41,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:14:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:41,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4204f23293292645e34a5d93616d8bf1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T07:14:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T07:14:41,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=A 2024-12-10T07:14:41,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:41,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=B 2024-12-10T07:14:41,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:41,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=C 2024-12-10T07:14:41,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:42,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T07:14:42,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:42,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:42,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:42,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:42,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:42,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/120f892f746441c788dd3754beb59ab6 is 50, key is test_row_0/A:col10/1733814881893/Put/seqid=0 2024-12-10T07:14:42,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:42,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814882044, max timestamp=1733814882093Num uniques 859; 2024-12-10T07:14:42,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:14:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59110 deadline: 1733814942089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:42,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:42,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814882044, max timestamp=1733814882093Num uniques 859; 2024-12-10T07:14:42,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:14:42,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814882044, max timestamp=1733814882093Num uniques 859; 2024-12-10T07:14:42,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:14:42,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741837_1013 (size=12001) 2024-12-10T07:14:42,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/120f892f746441c788dd3754beb59ab6 2024-12-10T07:14:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T07:14:42,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814942201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:42,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:42,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814942207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:42,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:42,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:42,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:42,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814942214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:42,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:42,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:42,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814942218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:42,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:42,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T07:14:42,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:42,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:42,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:42,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59110 deadline: 1733814942237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:42,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:42,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:42,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:42,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/8ed5f95145cd4dcab65c3728aa2d0340 is 50, key is test_row_0/B:col10/1733814881893/Put/seqid=0 2024-12-10T07:14:42,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741838_1014 (size=12001) 2024-12-10T07:14:42,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/8ed5f95145cd4dcab65c3728aa2d0340 2024-12-10T07:14:42,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814942326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:42,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814942326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:42,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:42,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814942327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:42,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:42,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:42,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814942335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:42,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:42,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/10fa3f411fdb4dfbabce05ebc4da3add is 50, key is test_row_0/C:col10/1733814881893/Put/seqid=0 2024-12-10T07:14:42,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741839_1015 (size=12001) 2024-12-10T07:14:42,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T07:14:42,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:42,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:42,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:42,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:42,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:42,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:42,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59110 deadline: 1733814942445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:42,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T07:14:42,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:42,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814942546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:42,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:42,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:42,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814942546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:42,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:42,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:42,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814942547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:42,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:42,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:42,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814942555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:42,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T07:14:42,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:42,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:42,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:42,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:42,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:42,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:42,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T07:14:42,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:42,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:42,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:42,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:42,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:42,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:42,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59110 deadline: 1733814942756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:42,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/10fa3f411fdb4dfbabce05ebc4da3add 2024-12-10T07:14:42,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/120f892f746441c788dd3754beb59ab6 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/120f892f746441c788dd3754beb59ab6 2024-12-10T07:14:42,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/120f892f746441c788dd3754beb59ab6, entries=150, sequenceid=12, filesize=11.7 K 2024-12-10T07:14:42,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/8ed5f95145cd4dcab65c3728aa2d0340 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/8ed5f95145cd4dcab65c3728aa2d0340 2024-12-10T07:14:42,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/8ed5f95145cd4dcab65c3728aa2d0340, entries=150, sequenceid=12, filesize=11.7 K 2024-12-10T07:14:42,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/10fa3f411fdb4dfbabce05ebc4da3add as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/10fa3f411fdb4dfbabce05ebc4da3add 2024-12-10T07:14:42,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/10fa3f411fdb4dfbabce05ebc4da3add, entries=150, sequenceid=12, filesize=11.7 K 2024-12-10T07:14:42,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4204f23293292645e34a5d93616d8bf1 in 912ms, sequenceid=12, compaction requested=false 2024-12-10T07:14:42,879 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-10T07:14:42,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:42,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-10T07:14:42,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:42,890 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 4204f23293292645e34a5d93616d8bf1 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-10T07:14:42,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=A 2024-12-10T07:14:42,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:14:42,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=B 2024-12-10T07:14:42,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:14:42,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=C 2024-12-10T07:14:42,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:14:42,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:42,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:42,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/fe4a3ec320bc4a838d4413a6028581bc is 50, key is test_row_0/A:col10/1733814882208/Put/seqid=0 2024-12-10T07:14:42,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:42,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814942931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:42,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:42,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:42,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741840_1016 (size=12001) 2024-12-10T07:14:42,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814942939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:42,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:42,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814942946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:42,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814942949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:42,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:42,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:43,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T07:14:43,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814943056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:43,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:43,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:43,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:43,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:43,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814943058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:43,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:43,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:43,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:43,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:43,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814943085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:43,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:43,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814943105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:43,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:43,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:43,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:43,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:43,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814943266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:43,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:43,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:43,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:43,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:43,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814943266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:43,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:43,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:43,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:43,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:43,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59110 deadline: 1733814943275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:43,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:43,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:43,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:43,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:43,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814943326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:43,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:43,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:43,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:43,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:43,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814943326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:43,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:43,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:43,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:43,352 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/fe4a3ec320bc4a838d4413a6028581bc 2024-12-10T07:14:43,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/14df0a8984c842d29827be370f1c0b5a is 50, key is test_row_0/B:col10/1733814882208/Put/seqid=0 2024-12-10T07:14:43,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741841_1017 (size=12001) 2024-12-10T07:14:43,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:43,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814943575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:43,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:43,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:43,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:43,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:43,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814943586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:43,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:43,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:43,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:43,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:43,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814943646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814943646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:43,826 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/14df0a8984c842d29827be370f1c0b5a 2024-12-10T07:14:43,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/a5bee603dde64bfdb34536aacdc1f700 is 50, key is test_row_0/C:col10/1733814882208/Put/seqid=0 2024-12-10T07:14:43,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741842_1018 (size=12001) 2024-12-10T07:14:43,878 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/a5bee603dde64bfdb34536aacdc1f700 2024-12-10T07:14:43,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/fe4a3ec320bc4a838d4413a6028581bc as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/fe4a3ec320bc4a838d4413a6028581bc 2024-12-10T07:14:43,916 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/fe4a3ec320bc4a838d4413a6028581bc, entries=150, sequenceid=47, filesize=11.7 K 2024-12-10T07:14:43,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/14df0a8984c842d29827be370f1c0b5a as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/14df0a8984c842d29827be370f1c0b5a 2024-12-10T07:14:43,939 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/14df0a8984c842d29827be370f1c0b5a, entries=150, sequenceid=47, filesize=11.7 K 2024-12-10T07:14:43,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/a5bee603dde64bfdb34536aacdc1f700 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/a5bee603dde64bfdb34536aacdc1f700 2024-12-10T07:14:43,961 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/a5bee603dde64bfdb34536aacdc1f700, entries=150, sequenceid=47, filesize=11.7 K 2024-12-10T07:14:43,964 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 4204f23293292645e34a5d93616d8bf1 in 1073ms, sequenceid=47, compaction requested=false 2024-12-10T07:14:43,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:43,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:43,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-10T07:14:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-10T07:14:43,973 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-10T07:14:43,974 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0890 sec 2024-12-10T07:14:43,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.1170 sec 2024-12-10T07:14:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-10T07:14:44,015 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:14:44,016 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:14:44,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-10T07:14:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T07:14:44,021 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:14:44,033 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:14:44,033 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:14:44,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:44,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4204f23293292645e34a5d93616d8bf1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T07:14:44,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=A 2024-12-10T07:14:44,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:44,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=B 2024-12-10T07:14:44,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:44,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=C 2024-12-10T07:14:44,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:44,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/5db5ac87b5154299abaf186999e6c217 is 50, key is test_row_0/A:col10/1733814884087/Put/seqid=0 2024-12-10T07:14:44,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T07:14:44,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T07:14:44,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:44,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:44,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:44,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:44,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:44,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:44,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741843_1019 (size=14341) 2024-12-10T07:14:44,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/5db5ac87b5154299abaf186999e6c217 2024-12-10T07:14:44,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/2159d9cd57c1479ab441feb3eb29c73a is 50, key is test_row_0/B:col10/1733814884087/Put/seqid=0 2024-12-10T07:14:44,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:44,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814944241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:44,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733814884219, max timestamp=1733814884247Num uniques 859; 2024-12-10T07:14:44,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T07:14:44,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:44,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741844_1020 (size=12001) 2024-12-10T07:14:44,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733814884219, max timestamp=1733814884247Num uniques 859; 2024-12-10T07:14:44,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T07:14:44,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733814884219, max timestamp=1733814884247Num uniques 859; 2024-12-10T07:14:44,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-10T07:14:44,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59110 deadline: 1733814944295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:44,309 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_2' in 4204f23293292645e34a5d93616d8bf1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:14:44,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:44,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814944305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:44,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:44,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:44,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814944311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:44,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:44,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814944311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T07:14:44,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T07:14:44,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:44,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:44,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:44,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:44,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:44,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:44,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814944355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:44,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:44,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814944425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:44,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814944425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:44,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814944425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:44,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T07:14:44,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:44,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:44,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:44,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:44,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:44,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:44,535 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T07:14:44,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814944570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:44,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:44,601 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-10T07:14:44,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814944635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:44,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:44,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814944635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:44,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:44,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814944636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:44,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:44,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T07:14:44,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/2159d9cd57c1479ab441feb3eb29c73a 2024-12-10T07:14:44,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T07:14:44,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:44,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:44,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:44,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:44,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:44,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/728b0308e00c41dbac804aeb70cad886 is 50, key is test_row_0/C:col10/1733814884087/Put/seqid=0 2024-12-10T07:14:44,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741845_1021 (size=12001) 2024-12-10T07:14:44,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T07:14:44,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:44,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:44,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:44,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:44,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:44,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:44,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:44,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814944885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:44,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:44,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814944945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:44,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:44,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814944945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:44,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:44,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814944955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:44,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:44,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:44,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T07:14:44,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:44,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:44,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:44,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:44,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:44,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:45,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/728b0308e00c41dbac804aeb70cad886 2024-12-10T07:14:45,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T07:14:45,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:45,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:45,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:45,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:45,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:45,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/5db5ac87b5154299abaf186999e6c217 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/5db5ac87b5154299abaf186999e6c217 2024-12-10T07:14:45,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T07:14:45,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/5db5ac87b5154299abaf186999e6c217, entries=200, sequenceid=59, filesize=14.0 K 2024-12-10T07:14:45,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/2159d9cd57c1479ab441feb3eb29c73a as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/2159d9cd57c1479ab441feb3eb29c73a 2024-12-10T07:14:45,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/2159d9cd57c1479ab441feb3eb29c73a, entries=150, sequenceid=59, filesize=11.7 K 2024-12-10T07:14:45,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/728b0308e00c41dbac804aeb70cad886 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/728b0308e00c41dbac804aeb70cad886 2024-12-10T07:14:45,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/728b0308e00c41dbac804aeb70cad886, entries=150, sequenceid=59, filesize=11.7 K 2024-12-10T07:14:45,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=118.13 KB/120965 for 4204f23293292645e34a5d93616d8bf1 in 1122ms, sequenceid=59, compaction requested=true 2024-12-10T07:14:45,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:45,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:14:45,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:45,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:14:45,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:14:45,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:14:45,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T07:14:45,222 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:14:45,223 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:14:45,227 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:14:45,229 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/B is initiating minor compaction (all files) 2024-12-10T07:14:45,229 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/B in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:45,230 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/8ed5f95145cd4dcab65c3728aa2d0340, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/14df0a8984c842d29827be370f1c0b5a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/2159d9cd57c1479ab441feb3eb29c73a] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=35.2 K 2024-12-10T07:14:45,232 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ed5f95145cd4dcab65c3728aa2d0340, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733814881893 2024-12-10T07:14:45,234 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 14df0a8984c842d29827be370f1c0b5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733814882201 2024-12-10T07:14:45,239 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 2159d9cd57c1479ab441feb3eb29c73a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733814882923 2024-12-10T07:14:45,245 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:14:45,253 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/A is initiating minor compaction (all files) 2024-12-10T07:14:45,253 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/A in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:45,253 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/120f892f746441c788dd3754beb59ab6, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/fe4a3ec320bc4a838d4413a6028581bc, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/5db5ac87b5154299abaf186999e6c217] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=37.4 K 2024-12-10T07:14:45,255 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 120f892f746441c788dd3754beb59ab6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733814881893 2024-12-10T07:14:45,256 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe4a3ec320bc4a838d4413a6028581bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733814882201 2024-12-10T07:14:45,265 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5db5ac87b5154299abaf186999e6c217, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733814882923 2024-12-10T07:14:45,287 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-10T07:14:45,288 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-10T07:14:45,290 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T07:14:45,290 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-10T07:14:45,290 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-10T07:14:45,290 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-10T07:14:45,291 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-10T07:14:45,291 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-10T07:14:45,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-10T07:14:45,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:45,307 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 4204f23293292645e34a5d93616d8bf1 3/3 column families, dataSize=118.13 KB heapSize=334.46 KB 2024-12-10T07:14:45,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=A 2024-12-10T07:14:45,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:14:45,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=B 2024-12-10T07:14:45,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:14:45,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=C 2024-12-10T07:14:45,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:14:45,318 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#B#compaction#9 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:14:45,319 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/df4db2c732d84fb49d47ea42d50080e6 is 50, key is test_row_0/B:col10/1733814884087/Put/seqid=0 2024-12-10T07:14:45,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741846_1022 (size=12104) 2024-12-10T07:14:45,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/76e07534eb0c44d7b474c1f684f0ae70 is 50, key is test_row_0/A:col10/1733814884301/Put/seqid=0 2024-12-10T07:14:45,379 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#A#compaction#11 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:14:45,380 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/05d96eba6bad4d44b69c7cfd9fb7806a is 50, key is test_row_0/A:col10/1733814884087/Put/seqid=0 2024-12-10T07:14:45,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741847_1023 (size=12001) 2024-12-10T07:14:45,399 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/76e07534eb0c44d7b474c1f684f0ae70 2024-12-10T07:14:45,409 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/df4db2c732d84fb49d47ea42d50080e6 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/df4db2c732d84fb49d47ea42d50080e6 2024-12-10T07:14:45,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:45,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:45,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/d3727846339e415799a49c358d2bf782 is 50, key is test_row_0/B:col10/1733814884301/Put/seqid=0 2024-12-10T07:14:45,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741848_1024 (size=12104) 2024-12-10T07:14:45,451 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/B of 4204f23293292645e34a5d93616d8bf1 into df4db2c732d84fb49d47ea42d50080e6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:14:45,452 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:45,471 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/B, priority=13, startTime=1733814885218; duration=0sec 2024-12-10T07:14:45,472 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:14:45,472 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:B 2024-12-10T07:14:45,472 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:14:45,473 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/05d96eba6bad4d44b69c7cfd9fb7806a as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/05d96eba6bad4d44b69c7cfd9fb7806a 2024-12-10T07:14:45,475 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:14:45,476 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/C is initiating minor compaction (all files) 2024-12-10T07:14:45,476 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/C in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:45,476 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/10fa3f411fdb4dfbabce05ebc4da3add, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/a5bee603dde64bfdb34536aacdc1f700, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/728b0308e00c41dbac804aeb70cad886] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=35.2 K 2024-12-10T07:14:45,477 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 10fa3f411fdb4dfbabce05ebc4da3add, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733814881893 2024-12-10T07:14:45,479 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting a5bee603dde64bfdb34536aacdc1f700, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733814882201 2024-12-10T07:14:45,481 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 728b0308e00c41dbac804aeb70cad886, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733814882923 2024-12-10T07:14:45,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741849_1025 (size=12001) 2024-12-10T07:14:45,505 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/A of 4204f23293292645e34a5d93616d8bf1 into 05d96eba6bad4d44b69c7cfd9fb7806a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:14:45,506 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:45,506 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/A, priority=13, startTime=1733814885213; duration=0sec 2024-12-10T07:14:45,506 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:45,506 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:A 2024-12-10T07:14:45,514 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#C#compaction#13 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:14:45,515 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/f6ecf8c294204d38ba7ce6c2668e8683 is 50, key is test_row_0/C:col10/1733814884087/Put/seqid=0 2024-12-10T07:14:45,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:45,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814945507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:45,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:45,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814945512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:45,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:45,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:45,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:45,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:45,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:45,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:45,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:45,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814945520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:45,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:45,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:45,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:45,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814945527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:45,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741850_1026 (size=12104) 2024-12-10T07:14:45,594 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/f6ecf8c294204d38ba7ce6c2668e8683 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/f6ecf8c294204d38ba7ce6c2668e8683 2024-12-10T07:14:45,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:45,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814945635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:45,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:45,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:45,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:45,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:45,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814945635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:45,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:45,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:45,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:45,650 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/C of 4204f23293292645e34a5d93616d8bf1 into f6ecf8c294204d38ba7ce6c2668e8683(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:14:45,650 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:45,650 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/C, priority=13, startTime=1733814885218; duration=0sec 2024-12-10T07:14:45,650 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:45,650 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:C 2024-12-10T07:14:45,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814945655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:45,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:45,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:45,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:45,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814945665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:45,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:45,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814945846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:45,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:45,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:45,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:45,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:45,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814945846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:45,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:45,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:45,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:45,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814945869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:45,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:45,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:45,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:45,895 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/d3727846339e415799a49c358d2bf782 2024-12-10T07:14:45,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814945885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:45,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:45,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:45,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:45,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/1ee66c102ef541d4a741c553dd138194 is 50, key is test_row_0/C:col10/1733814884301/Put/seqid=0 2024-12-10T07:14:46,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741851_1027 (size=12001) 2024-12-10T07:14:46,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814946156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:46,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:46,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:46,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:46,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T07:14:46,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:46,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814946166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:46,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:46,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:46,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:46,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:46,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814946185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:46,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:46,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:46,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:46,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:46,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814946225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:46,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:46,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59110 deadline: 1733814946325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:46,328 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_2' in 4204f23293292645e34a5d93616d8bf1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:14:46,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:46,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:46,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:46,422 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=37.62 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/1ee66c102ef541d4a741c553dd138194 2024-12-10T07:14:46,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/76e07534eb0c44d7b474c1f684f0ae70 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/76e07534eb0c44d7b474c1f684f0ae70 2024-12-10T07:14:46,480 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/76e07534eb0c44d7b474c1f684f0ae70, entries=150, sequenceid=93, filesize=11.7 K 2024-12-10T07:14:46,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/d3727846339e415799a49c358d2bf782 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/d3727846339e415799a49c358d2bf782 2024-12-10T07:14:46,507 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/d3727846339e415799a49c358d2bf782, entries=150, sequenceid=93, filesize=11.7 K 2024-12-10T07:14:46,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/1ee66c102ef541d4a741c553dd138194 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/1ee66c102ef541d4a741c553dd138194 2024-12-10T07:14:46,549 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/1ee66c102ef541d4a741c553dd138194, entries=150, sequenceid=93, filesize=11.7 K 2024-12-10T07:14:46,553 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~118.13 KB/120965, heapSize ~334.41 KB/342440, currentSize=67.09 KB/68700 for 4204f23293292645e34a5d93616d8bf1 in 1246ms, sequenceid=93, compaction requested=false 2024-12-10T07:14:46,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:46,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:46,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-10T07:14:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-10T07:14:46,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-10T07:14:46,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5320 sec 2024-12-10T07:14:46,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.5560 sec 2024-12-10T07:14:46,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:46,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4204f23293292645e34a5d93616d8bf1 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T07:14:46,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=A 2024-12-10T07:14:46,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:46,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=B 2024-12-10T07:14:46,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:46,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=C 2024-12-10T07:14:46,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:46,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/99249f607ae4473bb76969f33893314f is 50, key is test_row_0/A:col10/1733814886682/Put/seqid=0 2024-12-10T07:14:46,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741852_1028 (size=12001) 2024-12-10T07:14:46,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:46,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814946781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:46,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:46,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:46,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:46,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814946783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:46,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:46,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:46,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814946788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:46,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:46,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814946778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:46,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:46,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:46,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:46,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:46,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:46,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814946907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:46,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814946909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:46,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:46,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814946909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:46,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:46,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:46,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814946907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:47,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814947129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:47,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:47,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814947129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:47,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:47,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814947131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:47,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:47,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814947135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:47,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:47,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/99249f607ae4473bb76969f33893314f 2024-12-10T07:14:47,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/d3cfbba2edb7406e8f8efec283952945 is 50, key is test_row_0/B:col10/1733814886682/Put/seqid=0 2024-12-10T07:14:47,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741853_1029 (size=12001) 2024-12-10T07:14:47,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/d3cfbba2edb7406e8f8efec283952945 2024-12-10T07:14:47,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/a8602f8876a84bc3b1921b552477c446 is 50, key is test_row_0/C:col10/1733814886682/Put/seqid=0 2024-12-10T07:14:47,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741854_1030 (size=12001) 2024-12-10T07:14:47,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/a8602f8876a84bc3b1921b552477c446 2024-12-10T07:14:47,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/99249f607ae4473bb76969f33893314f as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/99249f607ae4473bb76969f33893314f 2024-12-10T07:14:47,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/99249f607ae4473bb76969f33893314f, entries=150, sequenceid=110, filesize=11.7 K 2024-12-10T07:14:47,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/d3cfbba2edb7406e8f8efec283952945 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/d3cfbba2edb7406e8f8efec283952945 2024-12-10T07:14:47,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/d3cfbba2edb7406e8f8efec283952945, entries=150, sequenceid=110, filesize=11.7 K 2024-12-10T07:14:47,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/a8602f8876a84bc3b1921b552477c446 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/a8602f8876a84bc3b1921b552477c446 2024-12-10T07:14:47,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/a8602f8876a84bc3b1921b552477c446, entries=150, sequenceid=110, filesize=11.7 K 2024-12-10T07:14:47,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 4204f23293292645e34a5d93616d8bf1 in 745ms, sequenceid=110, compaction requested=true 2024-12-10T07:14:47,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:47,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:14:47,433 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:14:47,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:47,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:14:47,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:14:47,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:14:47,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T07:14:47,434 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:14:47,436 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:14:47,436 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/A is initiating minor compaction (all files) 2024-12-10T07:14:47,436 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/A in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:47,436 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/05d96eba6bad4d44b69c7cfd9fb7806a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/76e07534eb0c44d7b474c1f684f0ae70, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/99249f607ae4473bb76969f33893314f] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=35.3 K 2024-12-10T07:14:47,437 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:14:47,437 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/B is initiating minor compaction (all files) 2024-12-10T07:14:47,437 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/B in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:47,441 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/df4db2c732d84fb49d47ea42d50080e6, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/d3727846339e415799a49c358d2bf782, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/d3cfbba2edb7406e8f8efec283952945] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=35.3 K 2024-12-10T07:14:47,441 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05d96eba6bad4d44b69c7cfd9fb7806a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733814882923 2024-12-10T07:14:47,442 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting df4db2c732d84fb49d47ea42d50080e6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733814882923 2024-12-10T07:14:47,442 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting d3727846339e415799a49c358d2bf782, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733814884284 2024-12-10T07:14:47,442 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76e07534eb0c44d7b474c1f684f0ae70, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733814884284 2024-12-10T07:14:47,443 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting d3cfbba2edb7406e8f8efec283952945, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733814885506 2024-12-10T07:14:47,444 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99249f607ae4473bb76969f33893314f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733814885506 2024-12-10T07:14:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:47,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4204f23293292645e34a5d93616d8bf1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T07:14:47,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=A 2024-12-10T07:14:47,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:47,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=B 2024-12-10T07:14:47,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:47,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=C 2024-12-10T07:14:47,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:47,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/08e8d09a450c46f3bf722a817b064582 is 50, key is test_row_0/A:col10/1733814887446/Put/seqid=0 2024-12-10T07:14:47,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741855_1031 (size=14391) 2024-12-10T07:14:47,478 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#B#compaction#19 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:14:47,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/08e8d09a450c46f3bf722a817b064582 2024-12-10T07:14:47,479 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#A#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:14:47,479 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/c21641fa50be4963ac25d1b03972e66e is 50, key is test_row_0/B:col10/1733814886682/Put/seqid=0 2024-12-10T07:14:47,479 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/8f0225770b004a8ebbf60893f4604d91 is 50, key is test_row_0/A:col10/1733814886682/Put/seqid=0 2024-12-10T07:14:47,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741856_1032 (size=12207) 2024-12-10T07:14:47,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741857_1033 (size=12207) 2024-12-10T07:14:47,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/8e5c3aebdae748688488418e6b264de0 is 50, key is test_row_0/B:col10/1733814887446/Put/seqid=0 2024-12-10T07:14:47,533 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/8f0225770b004a8ebbf60893f4604d91 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/8f0225770b004a8ebbf60893f4604d91 2024-12-10T07:14:47,546 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/c21641fa50be4963ac25d1b03972e66e as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/c21641fa50be4963ac25d1b03972e66e 2024-12-10T07:14:47,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741858_1034 (size=12051) 2024-12-10T07:14:47,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/8e5c3aebdae748688488418e6b264de0 2024-12-10T07:14:47,589 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/B of 4204f23293292645e34a5d93616d8bf1 into c21641fa50be4963ac25d1b03972e66e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:14:47,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/4dd51fc60b8b41928e8475639c9eec8c is 50, key is test_row_0/C:col10/1733814887446/Put/seqid=0 2024-12-10T07:14:47,589 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:47,590 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/B, priority=13, startTime=1733814887433; duration=0sec 2024-12-10T07:14:47,590 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:14:47,590 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:B 2024-12-10T07:14:47,590 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:14:47,591 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/A of 4204f23293292645e34a5d93616d8bf1 into 8f0225770b004a8ebbf60893f4604d91(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:14:47,591 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:47,591 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/A, priority=13, startTime=1733814887433; duration=0sec 2024-12-10T07:14:47,592 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:47,592 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:A 2024-12-10T07:14:47,597 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:14:47,597 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/C is initiating minor compaction (all files) 2024-12-10T07:14:47,597 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/C in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:47,597 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/f6ecf8c294204d38ba7ce6c2668e8683, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/1ee66c102ef541d4a741c553dd138194, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/a8602f8876a84bc3b1921b552477c446] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=35.3 K 2024-12-10T07:14:47,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814947576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:47,605 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting f6ecf8c294204d38ba7ce6c2668e8683, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733814882923 2024-12-10T07:14:47,606 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ee66c102ef541d4a741c553dd138194, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733814884284 2024-12-10T07:14:47,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814947588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:47,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:47,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:47,612 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting a8602f8876a84bc3b1921b552477c446, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733814885506 2024-12-10T07:14:47,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814947590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:47,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:47,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814947594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:47,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:47,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741859_1035 (size=12051) 2024-12-10T07:14:47,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/4dd51fc60b8b41928e8475639c9eec8c 2024-12-10T07:14:47,663 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#C#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:14:47,664 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/4c8c387b95c9486fabff3915d6d39777 is 50, key is test_row_0/C:col10/1733814886682/Put/seqid=0 2024-12-10T07:14:47,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/08e8d09a450c46f3bf722a817b064582 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/08e8d09a450c46f3bf722a817b064582 2024-12-10T07:14:47,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/08e8d09a450c46f3bf722a817b064582, entries=200, sequenceid=129, filesize=14.1 K 2024-12-10T07:14:47,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/8e5c3aebdae748688488418e6b264de0 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/8e5c3aebdae748688488418e6b264de0 2024-12-10T07:14:47,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/8e5c3aebdae748688488418e6b264de0, entries=150, sequenceid=129, filesize=11.8 K 2024-12-10T07:14:47,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/4dd51fc60b8b41928e8475639c9eec8c as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/4dd51fc60b8b41928e8475639c9eec8c 2024-12-10T07:14:47,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741860_1036 (size=12207) 2024-12-10T07:14:47,731 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/4c8c387b95c9486fabff3915d6d39777 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/4c8c387b95c9486fabff3915d6d39777 2024-12-10T07:14:47,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/4dd51fc60b8b41928e8475639c9eec8c, entries=150, sequenceid=129, filesize=11.8 K 2024-12-10T07:14:47,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 4204f23293292645e34a5d93616d8bf1 in 288ms, sequenceid=129, compaction requested=false 2024-12-10T07:14:47,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:47,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4204f23293292645e34a5d93616d8bf1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T07:14:47,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=A 2024-12-10T07:14:47,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:47,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=B 2024-12-10T07:14:47,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:47,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=C 2024-12-10T07:14:47,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:47,748 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/C of 4204f23293292645e34a5d93616d8bf1 into 4c8c387b95c9486fabff3915d6d39777(size=11.9 K), total size for store is 23.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:14:47,748 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:47,748 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/C, priority=13, startTime=1733814887433; duration=0sec 2024-12-10T07:14:47,749 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:47,749 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:C 2024-12-10T07:14:47,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/bc32f62739514cfd9ddeb393f2a0935a is 50, key is test_row_0/A:col10/1733814887533/Put/seqid=0 2024-12-10T07:14:47,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741861_1037 (size=16931) 2024-12-10T07:14:47,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/bc32f62739514cfd9ddeb393f2a0935a 2024-12-10T07:14:47,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:47,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/f0efd937b6a24c299784aaea83c8d868 is 50, key is test_row_0/B:col10/1733814887533/Put/seqid=0 2024-12-10T07:14:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814947809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:47,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:47,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814947817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:47,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:47,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814947825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:47,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814947827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:47,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:47,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741862_1038 (size=12151) 2024-12-10T07:14:47,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/f0efd937b6a24c299784aaea83c8d868 2024-12-10T07:14:47,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/367d918a5d834d5e9acb4fead9c291a1 is 50, key is test_row_0/C:col10/1733814887533/Put/seqid=0 2024-12-10T07:14:47,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:47,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814947935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:47,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:47,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741863_1039 (size=12151) 2024-12-10T07:14:47,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814947945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:47,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:47,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:47,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814947946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:47,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:47,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814947946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:48,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814948145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:48,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:48,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:48,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:48,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:48,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814948166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:48,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:48,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:48,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:48,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:48,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814948167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:48,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:48,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:48,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:48,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814948167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:48,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:48,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:48,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-10T07:14:48,195 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:14:48,196 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:14:48,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-10T07:14:48,201 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:14:48,202 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:14:48,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:14:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T07:14:48,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T07:14:48,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/367d918a5d834d5e9acb4fead9c291a1 2024-12-10T07:14:48,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-10T07:14:48,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:48,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:48,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:48,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:48,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:48,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:48,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/bc32f62739514cfd9ddeb393f2a0935a as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/bc32f62739514cfd9ddeb393f2a0935a 2024-12-10T07:14:48,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/bc32f62739514cfd9ddeb393f2a0935a, entries=250, sequenceid=147, filesize=16.5 K 2024-12-10T07:14:48,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/f0efd937b6a24c299784aaea83c8d868 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/f0efd937b6a24c299784aaea83c8d868 2024-12-10T07:14:48,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814948469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:48,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:48,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:48,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:48,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:48,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814948475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:48,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:48,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:48,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:48,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:48,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/f0efd937b6a24c299784aaea83c8d868, entries=150, sequenceid=147, filesize=11.9 K 2024-12-10T07:14:48,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814948486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:48,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:48,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:48,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:48,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814948486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:48,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:48,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:48,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:48,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/367d918a5d834d5e9acb4fead9c291a1 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/367d918a5d834d5e9acb4fead9c291a1 2024-12-10T07:14:48,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/367d918a5d834d5e9acb4fead9c291a1, entries=150, sequenceid=147, filesize=11.9 K 2024-12-10T07:14:48,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 4204f23293292645e34a5d93616d8bf1 in 779ms, sequenceid=147, compaction requested=true 2024-12-10T07:14:48,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:48,518 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:14:48,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:14:48,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:48,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:14:48,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:14:48,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:14:48,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-10T07:14:48,521 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43529 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:14:48,521 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/A is initiating minor compaction (all files) 2024-12-10T07:14:48,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:48,521 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/A in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:48,521 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/8f0225770b004a8ebbf60893f4604d91, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/08e8d09a450c46f3bf722a817b064582, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/bc32f62739514cfd9ddeb393f2a0935a] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=42.5 K 2024-12-10T07:14:48,521 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 4204f23293292645e34a5d93616d8bf1 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T07:14:48,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=A 2024-12-10T07:14:48,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:48,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=B 2024-12-10T07:14:48,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:48,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=C 2024-12-10T07:14:48,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:48,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T07:14:48,524 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:14:48,524 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f0225770b004a8ebbf60893f4604d91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733814885506 2024-12-10T07:14:48,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T07:14:48,527 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 08e8d09a450c46f3bf722a817b064582, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733814886772 2024-12-10T07:14:48,529 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting bc32f62739514cfd9ddeb393f2a0935a, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733814887533 2024-12-10T07:14:48,532 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:14:48,532 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/B is initiating minor compaction (all files) 2024-12-10T07:14:48,532 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/B in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:48,532 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/c21641fa50be4963ac25d1b03972e66e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/8e5c3aebdae748688488418e6b264de0, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/f0efd937b6a24c299784aaea83c8d868] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=35.6 K 2024-12-10T07:14:48,534 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting c21641fa50be4963ac25d1b03972e66e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733814885506 2024-12-10T07:14:48,535 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e5c3aebdae748688488418e6b264de0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733814886778 2024-12-10T07:14:48,536 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0efd937b6a24c299784aaea83c8d868, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733814887533 2024-12-10T07:14:48,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/388085c85a494a168a52ce60fedb3d54 is 50, key is test_row_0/A:col10/1733814887790/Put/seqid=0 2024-12-10T07:14:48,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741864_1040 (size=12151) 2024-12-10T07:14:48,565 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/388085c85a494a168a52ce60fedb3d54 2024-12-10T07:14:48,581 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#A#compaction#28 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:14:48,582 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/30d40f2ce79c43ff9a3714077a9cc445 is 50, key is test_row_0/A:col10/1733814887533/Put/seqid=0 2024-12-10T07:14:48,587 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#B#compaction#29 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:14:48,587 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/54782b43d2a64f62b9738d9af0b1f8ff is 50, key is test_row_0/B:col10/1733814887533/Put/seqid=0 2024-12-10T07:14:48,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/437efa10cf3a461b9d33b620c2e7d435 is 50, key is test_row_0/B:col10/1733814887790/Put/seqid=0 2024-12-10T07:14:48,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741865_1041 (size=12459) 2024-12-10T07:14:48,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741866_1042 (size=12459) 2024-12-10T07:14:48,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741867_1043 (size=12151) 2024-12-10T07:14:48,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T07:14:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:48,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:49,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814949022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:49,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:49,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814949027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:49,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:49,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:49,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814949023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:49,041 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/30d40f2ce79c43ff9a3714077a9cc445 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/30d40f2ce79c43ff9a3714077a9cc445 2024-12-10T07:14:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:49,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:49,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814949029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:49,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:49,070 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/437efa10cf3a461b9d33b620c2e7d435 2024-12-10T07:14:49,072 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/54782b43d2a64f62b9738d9af0b1f8ff as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/54782b43d2a64f62b9738d9af0b1f8ff 2024-12-10T07:14:49,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/ae6bb01ff79a46ee8b2a896dde96677e is 50, key is test_row_0/C:col10/1733814887790/Put/seqid=0 2024-12-10T07:14:49,109 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/A of 4204f23293292645e34a5d93616d8bf1 into 30d40f2ce79c43ff9a3714077a9cc445(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:14:49,109 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:49,109 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/A, priority=13, startTime=1733814888518; duration=0sec 2024-12-10T07:14:49,110 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:14:49,110 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:A 2024-12-10T07:14:49,110 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:14:49,111 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/B of 4204f23293292645e34a5d93616d8bf1 into 54782b43d2a64f62b9738d9af0b1f8ff(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:14:49,111 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:49,111 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/B, priority=13, startTime=1733814888518; duration=0sec 2024-12-10T07:14:49,111 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:49,111 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:B 2024-12-10T07:14:49,115 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:14:49,115 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/C is initiating minor compaction (all files) 2024-12-10T07:14:49,116 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/C in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:49,116 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/4c8c387b95c9486fabff3915d6d39777, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/4dd51fc60b8b41928e8475639c9eec8c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/367d918a5d834d5e9acb4fead9c291a1] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=35.6 K 2024-12-10T07:14:49,117 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c8c387b95c9486fabff3915d6d39777, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733814885506 2024-12-10T07:14:49,118 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dd51fc60b8b41928e8475639c9eec8c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733814886778 2024-12-10T07:14:49,119 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 367d918a5d834d5e9acb4fead9c291a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733814887533 2024-12-10T07:14:49,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:49,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814949146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:49,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:49,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814949146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:49,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741868_1044 (size=12151) 2024-12-10T07:14:49,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:49,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:49,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814949146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:49,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:49,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:49,154 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#C#compaction#32 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:14:49,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:49,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814949155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:49,162 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/ae6bb01ff79a46ee8b2a896dde96677e 2024-12-10T07:14:49,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:49,156 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/4ff7252a261b4db694c16b8501d4f79a is 50, key is test_row_0/C:col10/1733814887533/Put/seqid=0 2024-12-10T07:14:49,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/388085c85a494a168a52ce60fedb3d54 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/388085c85a494a168a52ce60fedb3d54 2024-12-10T07:14:49,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741869_1045 (size=12459) 2024-12-10T07:14:49,232 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/388085c85a494a168a52ce60fedb3d54, entries=150, sequenceid=165, filesize=11.9 K 2024-12-10T07:14:49,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/437efa10cf3a461b9d33b620c2e7d435 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/437efa10cf3a461b9d33b620c2e7d435 2024-12-10T07:14:49,252 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/4ff7252a261b4db694c16b8501d4f79a as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/4ff7252a261b4db694c16b8501d4f79a 2024-12-10T07:14:49,273 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/437efa10cf3a461b9d33b620c2e7d435, entries=150, sequenceid=165, filesize=11.9 K 2024-12-10T07:14:49,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/ae6bb01ff79a46ee8b2a896dde96677e as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/ae6bb01ff79a46ee8b2a896dde96677e 2024-12-10T07:14:49,293 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/C of 4204f23293292645e34a5d93616d8bf1 into 4ff7252a261b4db694c16b8501d4f79a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:14:49,293 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:49,294 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/C, priority=13, startTime=1733814888519; duration=0sec 2024-12-10T07:14:49,294 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:49,294 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:C 2024-12-10T07:14:49,300 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/ae6bb01ff79a46ee8b2a896dde96677e, entries=150, sequenceid=165, filesize=11.9 K 2024-12-10T07:14:49,305 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4204f23293292645e34a5d93616d8bf1 in 784ms, sequenceid=165, compaction requested=false 2024-12-10T07:14:49,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:49,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:49,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-10T07:14:49,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-10T07:14:49,321 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-10T07:14:49,322 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1130 sec 2024-12-10T07:14:49,326 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.1270 sec 2024-12-10T07:14:49,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-10T07:14:49,357 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:14:49,358 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:14:49,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-10T07:14:49,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T07:14:49,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:49,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4204f23293292645e34a5d93616d8bf1 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T07:14:49,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=A 2024-12-10T07:14:49,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:49,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=B 2024-12-10T07:14:49,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:49,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=C 2024-12-10T07:14:49,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:49,377 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:14:49,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/21eeadb2f43446f2a5468d5d4b8695db is 50, key is test_row_0/A:col10/1733814889020/Put/seqid=0 2024-12-10T07:14:49,385 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:14:49,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:14:49,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:49,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814949399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:49,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814949418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:49,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:49,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:49,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814949418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:49,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:49,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:49,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814949419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:49,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T07:14:49,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741870_1046 (size=12151) 2024-12-10T07:14:49,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814949525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:49,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:49,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:49,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814949535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:49,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:49,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:49,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814949537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814949537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:49,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T07:14:49,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:49,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:49,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:49,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:49,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:49,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:49,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:49,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T07:14:49,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T07:14:49,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:49,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:49,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:49,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:49,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:49,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:49,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814949734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:49,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:49,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814949745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814949745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:49,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:49,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:49,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814949754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:49,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:49,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:49,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T07:14:49,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:49,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:49,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:49,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:49,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:49,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/21eeadb2f43446f2a5468d5d4b8695db 2024-12-10T07:14:49,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/5d681fa896f0453d95df6ad0d7ea5e90 is 50, key is test_row_0/B:col10/1733814889020/Put/seqid=0 2024-12-10T07:14:49,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741871_1047 (size=12151) 2024-12-10T07:14:49,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/5d681fa896f0453d95df6ad0d7ea5e90 2024-12-10T07:14:49,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/6877685e2960444186c6bb2e9e48f3ab is 50, key is test_row_0/C:col10/1733814889020/Put/seqid=0 2024-12-10T07:14:49,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T07:14:49,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741872_1048 (size=12151) 2024-12-10T07:14:50,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T07:14:50,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:50,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:50,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:50,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:50,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:50,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:50,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814950045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:50,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:50,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:50,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814950055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:50,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814950057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:50,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:50,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:50,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814950066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:50,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:50,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T07:14:50,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:50,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:50,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:50,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:50,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:50,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:50,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:50,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59110 deadline: 1733814950355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:50,360 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_2' in 4204f23293292645e34a5d93616d8bf1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8272 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:14:50,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:50,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T07:14:50,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:50,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:50,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:50,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:50,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:50,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:50,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/6877685e2960444186c6bb2e9e48f3ab 2024-12-10T07:14:50,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/21eeadb2f43446f2a5468d5d4b8695db as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/21eeadb2f43446f2a5468d5d4b8695db 2024-12-10T07:14:50,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/21eeadb2f43446f2a5468d5d4b8695db, entries=150, sequenceid=185, filesize=11.9 K 2024-12-10T07:14:50,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/5d681fa896f0453d95df6ad0d7ea5e90 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/5d681fa896f0453d95df6ad0d7ea5e90 2024-12-10T07:14:50,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/5d681fa896f0453d95df6ad0d7ea5e90, entries=150, sequenceid=185, filesize=11.9 K 2024-12-10T07:14:50,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/6877685e2960444186c6bb2e9e48f3ab as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/6877685e2960444186c6bb2e9e48f3ab 2024-12-10T07:14:50,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T07:14:50,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/6877685e2960444186c6bb2e9e48f3ab, entries=150, sequenceid=185, filesize=11.9 K 2024-12-10T07:14:50,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 4204f23293292645e34a5d93616d8bf1 in 1152ms, sequenceid=185, compaction requested=true 2024-12-10T07:14:50,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:50,521 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:14:50,523 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:14:50,523 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/A is initiating minor compaction (all files) 2024-12-10T07:14:50,523 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/A in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:50,524 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/30d40f2ce79c43ff9a3714077a9cc445, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/388085c85a494a168a52ce60fedb3d54, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/21eeadb2f43446f2a5468d5d4b8695db] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=35.9 K 2024-12-10T07:14:50,524 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30d40f2ce79c43ff9a3714077a9cc445, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733814887533 2024-12-10T07:14:50,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:14:50,525 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 388085c85a494a168a52ce60fedb3d54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733814887786 2024-12-10T07:14:50,526 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21eeadb2f43446f2a5468d5d4b8695db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733814889020 2024-12-10T07:14:50,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:50,528 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:14:50,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:14:50,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:50,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:14:50,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:14:50,530 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:14:50,530 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/B is initiating minor compaction (all files) 2024-12-10T07:14:50,531 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/B in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:50,531 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/54782b43d2a64f62b9738d9af0b1f8ff, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/437efa10cf3a461b9d33b620c2e7d435, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/5d681fa896f0453d95df6ad0d7ea5e90] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=35.9 K 2024-12-10T07:14:50,531 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 54782b43d2a64f62b9738d9af0b1f8ff, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733814887533 2024-12-10T07:14:50,532 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 437efa10cf3a461b9d33b620c2e7d435, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733814887786 2024-12-10T07:14:50,532 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d681fa896f0453d95df6ad0d7ea5e90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733814889020 2024-12-10T07:14:50,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-10T07:14:50,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:50,541 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 4204f23293292645e34a5d93616d8bf1 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T07:14:50,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=A 2024-12-10T07:14:50,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:50,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=B 2024-12-10T07:14:50,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:50,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=C 2024-12-10T07:14:50,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:50,551 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#B#compaction#36 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:14:50,552 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/4e95020c77f64cb9810487c1c2fb5e94 is 50, key is test_row_0/B:col10/1733814889020/Put/seqid=0 2024-12-10T07:14:50,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:50,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:50,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/85cf3432970d43bbad6a098de4e520d9 is 50, key is test_row_0/A:col10/1733814889407/Put/seqid=0 2024-12-10T07:14:50,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741873_1049 (size=12561) 2024-12-10T07:14:50,588 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#A#compaction#38 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:14:50,588 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/52f80f7931594ac7b3cb7103f90ed548 is 50, key is test_row_0/A:col10/1733814889020/Put/seqid=0 2024-12-10T07:14:50,599 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/4e95020c77f64cb9810487c1c2fb5e94 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/4e95020c77f64cb9810487c1c2fb5e94 2024-12-10T07:14:50,612 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/B of 4204f23293292645e34a5d93616d8bf1 into 4e95020c77f64cb9810487c1c2fb5e94(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:14:50,613 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:50,613 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/B, priority=13, startTime=1733814890528; duration=0sec 2024-12-10T07:14:50,614 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:14:50,614 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:B 2024-12-10T07:14:50,614 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:14:50,622 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:14:50,623 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/C is initiating minor compaction (all files) 2024-12-10T07:14:50,623 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/C in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:50,623 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/4ff7252a261b4db694c16b8501d4f79a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/ae6bb01ff79a46ee8b2a896dde96677e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/6877685e2960444186c6bb2e9e48f3ab] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=35.9 K 2024-12-10T07:14:50,624 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ff7252a261b4db694c16b8501d4f79a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733814887533 2024-12-10T07:14:50,624 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting ae6bb01ff79a46ee8b2a896dde96677e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733814887786 2024-12-10T07:14:50,625 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 6877685e2960444186c6bb2e9e48f3ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733814889020 2024-12-10T07:14:50,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741874_1050 (size=12151) 2024-12-10T07:14:50,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741875_1051 (size=12561) 2024-12-10T07:14:50,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:50,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814950618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:50,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:50,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:50,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814950619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:50,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:50,645 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/85cf3432970d43bbad6a098de4e520d9 2024-12-10T07:14:50,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:50,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814950622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:50,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:50,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814950620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:50,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:50,668 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#C#compaction#39 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:14:50,669 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/cf46de35e7214f28a884f3e8ce19afa0 is 50, key is test_row_0/C:col10/1733814889020/Put/seqid=0 2024-12-10T07:14:50,687 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/52f80f7931594ac7b3cb7103f90ed548 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/52f80f7931594ac7b3cb7103f90ed548 2024-12-10T07:14:50,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741876_1052 (size=12561) 2024-12-10T07:14:50,712 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/A of 4204f23293292645e34a5d93616d8bf1 into 52f80f7931594ac7b3cb7103f90ed548(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:14:50,713 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:50,713 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/A, priority=13, startTime=1733814890521; duration=0sec 2024-12-10T07:14:50,713 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:50,713 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:A 2024-12-10T07:14:50,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/a2bf2f37c0ab4621a42691bb4fb0444b is 50, key is test_row_0/B:col10/1733814889407/Put/seqid=0 2024-12-10T07:14:50,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:50,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814950757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814950750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:50,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:50,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814950757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:50,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:50,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:50,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741877_1053 (size=12151) 2024-12-10T07:14:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814950758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:50,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:50,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:50,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814950985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:50,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:50,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814950985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:50,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:50,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:50,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814950985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:50,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:50,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:50,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:50,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:51,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814950989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:51,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:51,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:51,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:51,119 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/cf46de35e7214f28a884f3e8ce19afa0 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/cf46de35e7214f28a884f3e8ce19afa0 2024-12-10T07:14:51,140 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/C of 4204f23293292645e34a5d93616d8bf1 into cf46de35e7214f28a884f3e8ce19afa0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:14:51,141 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:51,141 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/C, priority=13, startTime=1733814890529; duration=0sec 2024-12-10T07:14:51,141 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:51,141 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:C 2024-12-10T07:14:51,176 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/a2bf2f37c0ab4621a42691bb4fb0444b 2024-12-10T07:14:51,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/f1c8732544d74f4f8f0f1dacb44bbdc4 is 50, key is test_row_0/C:col10/1733814889407/Put/seqid=0 2024-12-10T07:14:51,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741878_1054 (size=12151) 2024-12-10T07:14:51,235 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/f1c8732544d74f4f8f0f1dacb44bbdc4 2024-12-10T07:14:51,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/85cf3432970d43bbad6a098de4e520d9 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/85cf3432970d43bbad6a098de4e520d9 2024-12-10T07:14:51,269 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/85cf3432970d43bbad6a098de4e520d9, entries=150, sequenceid=201, filesize=11.9 K 2024-12-10T07:14:51,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/a2bf2f37c0ab4621a42691bb4fb0444b as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/a2bf2f37c0ab4621a42691bb4fb0444b 2024-12-10T07:14:51,299 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/a2bf2f37c0ab4621a42691bb4fb0444b, entries=150, sequenceid=201, filesize=11.9 K 2024-12-10T07:14:51,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/f1c8732544d74f4f8f0f1dacb44bbdc4 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/f1c8732544d74f4f8f0f1dacb44bbdc4 2024-12-10T07:14:51,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814951305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:51,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814951306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:51,316 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/f1c8732544d74f4f8f0f1dacb44bbdc4, entries=150, sequenceid=201, filesize=11.9 K 2024-12-10T07:14:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:51,322 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 4204f23293292645e34a5d93616d8bf1 in 781ms, sequenceid=201, compaction requested=false 2024-12-10T07:14:51,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:51,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:51,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-10T07:14:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:51,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4204f23293292645e34a5d93616d8bf1 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T07:14:51,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=A 2024-12-10T07:14:51,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:51,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=B 2024-12-10T07:14:51,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:51,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=C 2024-12-10T07:14:51,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:51,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:51,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:51,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:51,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-10T07:14:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814951305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:51,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:51,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:51,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:51,361 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-10T07:14:51,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/e312cbbd5cb140f0ae1977310517cc1e is 50, key is test_row_0/A:col10/1733814890619/Put/seqid=0 2024-12-10T07:14:51,362 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9720 sec 2024-12-10T07:14:51,369 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.0060 sec 2024-12-10T07:14:51,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741879_1055 (size=14541) 2024-12-10T07:14:51,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/e312cbbd5cb140f0ae1977310517cc1e 2024-12-10T07:14:51,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/bcafcb93905b4881abe865cbd0c22230 is 50, key is test_row_0/B:col10/1733814890619/Put/seqid=0 2024-12-10T07:14:51,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741880_1056 (size=12151) 2024-12-10T07:14:51,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/bcafcb93905b4881abe865cbd0c22230 2024-12-10T07:14:51,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/e20d09f87a344bb5a65b63d92d9c4bca is 50, key is test_row_0/C:col10/1733814890619/Put/seqid=0 2024-12-10T07:14:51,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814951485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:51,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:51,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:51,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-10T07:14:51,526 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:14:51,527 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:14:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-10T07:14:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T07:14:51,531 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:14:51,533 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:14:51,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:14:51,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741881_1057 (size=12151) 2024-12-10T07:14:51,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/e20d09f87a344bb5a65b63d92d9c4bca 2024-12-10T07:14:51,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/e312cbbd5cb140f0ae1977310517cc1e as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/e312cbbd5cb140f0ae1977310517cc1e 2024-12-10T07:14:51,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/e312cbbd5cb140f0ae1977310517cc1e, entries=200, sequenceid=221, filesize=14.2 K 2024-12-10T07:14:51,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/bcafcb93905b4881abe865cbd0c22230 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/bcafcb93905b4881abe865cbd0c22230 2024-12-10T07:14:51,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814951595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:51,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:51,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:51,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:51,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/bcafcb93905b4881abe865cbd0c22230, entries=150, sequenceid=221, filesize=11.9 K 2024-12-10T07:14:51,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/e20d09f87a344bb5a65b63d92d9c4bca as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/e20d09f87a344bb5a65b63d92d9c4bca 2024-12-10T07:14:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T07:14:51,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/e20d09f87a344bb5a65b63d92d9c4bca, entries=150, sequenceid=221, filesize=11.9 K 2024-12-10T07:14:51,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4204f23293292645e34a5d93616d8bf1 in 334ms, sequenceid=221, compaction requested=true 2024-12-10T07:14:51,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:51,658 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:14:51,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:14:51,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:51,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:14:51,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:51,658 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:14:51,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:14:51,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:14:51,661 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:14:51,661 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/A is initiating minor compaction (all files) 2024-12-10T07:14:51,662 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/A in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:51,662 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/52f80f7931594ac7b3cb7103f90ed548, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/85cf3432970d43bbad6a098de4e520d9, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/e312cbbd5cb140f0ae1977310517cc1e] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=38.3 K 2024-12-10T07:14:51,663 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:14:51,663 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/B is initiating minor compaction (all files) 2024-12-10T07:14:51,663 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/B in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:51,663 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/4e95020c77f64cb9810487c1c2fb5e94, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/a2bf2f37c0ab4621a42691bb4fb0444b, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/bcafcb93905b4881abe865cbd0c22230] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=36.0 K 2024-12-10T07:14:51,663 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52f80f7931594ac7b3cb7103f90ed548, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733814889020 2024-12-10T07:14:51,664 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85cf3432970d43bbad6a098de4e520d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733814889378 2024-12-10T07:14:51,664 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e95020c77f64cb9810487c1c2fb5e94, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733814889020 2024-12-10T07:14:51,665 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting e312cbbd5cb140f0ae1977310517cc1e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733814890614 2024-12-10T07:14:51,665 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting a2bf2f37c0ab4621a42691bb4fb0444b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733814889378 2024-12-10T07:14:51,667 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting bcafcb93905b4881abe865cbd0c22230, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733814890617 2024-12-10T07:14:51,688 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#B#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:14:51,689 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/d4bb173a0c28475a927b8f4c6b6dcc0e is 50, key is test_row_0/B:col10/1733814890619/Put/seqid=0 2024-12-10T07:14:51,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-10T07:14:51,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:51,698 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 4204f23293292645e34a5d93616d8bf1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T07:14:51,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=A 2024-12-10T07:14:51,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:51,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=B 2024-12-10T07:14:51,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:51,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=C 2024-12-10T07:14:51,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:51,699 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#A#compaction#46 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:14:51,700 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/1eb803766bf74c0da10f88eb02c2bafe is 50, key is test_row_0/A:col10/1733814890619/Put/seqid=0 2024-12-10T07:14:51,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/8a3c30f62d434009800efe8d86e583b6 is 50, key is test_row_0/A:col10/1733814891439/Put/seqid=0 2024-12-10T07:14:51,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741882_1058 (size=12663) 2024-12-10T07:14:51,774 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/d4bb173a0c28475a927b8f4c6b6dcc0e as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/d4bb173a0c28475a927b8f4c6b6dcc0e 2024-12-10T07:14:51,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741883_1059 (size=12663) 2024-12-10T07:14:51,790 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/B of 4204f23293292645e34a5d93616d8bf1 into d4bb173a0c28475a927b8f4c6b6dcc0e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:14:51,790 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:51,790 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/B, priority=13, startTime=1733814891658; duration=0sec 2024-12-10T07:14:51,790 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:14:51,790 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:B 2024-12-10T07:14:51,790 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:14:51,793 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/1eb803766bf74c0da10f88eb02c2bafe as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/1eb803766bf74c0da10f88eb02c2bafe 2024-12-10T07:14:51,796 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:14:51,796 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/C is initiating minor compaction (all files) 2024-12-10T07:14:51,796 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/C in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:51,797 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/cf46de35e7214f28a884f3e8ce19afa0, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/f1c8732544d74f4f8f0f1dacb44bbdc4, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/e20d09f87a344bb5a65b63d92d9c4bca] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=36.0 K 2024-12-10T07:14:51,801 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting cf46de35e7214f28a884f3e8ce19afa0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733814889020 2024-12-10T07:14:51,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741884_1060 (size=12151) 2024-12-10T07:14:51,804 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting f1c8732544d74f4f8f0f1dacb44bbdc4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733814889378 2024-12-10T07:14:51,813 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting e20d09f87a344bb5a65b63d92d9c4bca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733814890617 2024-12-10T07:14:51,814 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/A of 4204f23293292645e34a5d93616d8bf1 into 1eb803766bf74c0da10f88eb02c2bafe(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:14:51,815 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:51,815 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/A, priority=13, startTime=1733814891657; duration=0sec 2024-12-10T07:14:51,815 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:51,815 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:A 2024-12-10T07:14:51,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:51,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T07:14:51,860 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#C#compaction#48 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:14:51,860 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/07929e3f1a1b45c5b76aa271be9c0920 is 50, key is test_row_0/C:col10/1733814890619/Put/seqid=0 2024-12-10T07:14:51,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814951886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:51,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814951887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:51,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:51,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:51,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814951891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:51,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:51,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:51,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:51,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814951893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:51,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741885_1061 (size=12663) 2024-12-10T07:14:52,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814952005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:52,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:52,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:52,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814952006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:52,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:52,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814952006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814952005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:52,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:52,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:52,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:52,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:52,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:52,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:52,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:52,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:52,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T07:14:52,206 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/8a3c30f62d434009800efe8d86e583b6 2024-12-10T07:14:52,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814952215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:52,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814952217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:52,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:52,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:52,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:52,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814952225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:52,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:52,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:52,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:52,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814952230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:52,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/8bd2cf5c7c984a40b2b575e50489b515 is 50, key is test_row_0/B:col10/1733814891439/Put/seqid=0 2024-12-10T07:14:52,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:52,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:52,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:52,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741886_1062 (size=12151) 2024-12-10T07:14:52,269 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/8bd2cf5c7c984a40b2b575e50489b515 2024-12-10T07:14:52,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/299b8c6b31a74d21a61e95845a5453b5 is 50, key is test_row_0/C:col10/1733814891439/Put/seqid=0 2024-12-10T07:14:52,327 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/07929e3f1a1b45c5b76aa271be9c0920 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/07929e3f1a1b45c5b76aa271be9c0920 2024-12-10T07:14:52,345 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/C of 4204f23293292645e34a5d93616d8bf1 into 07929e3f1a1b45c5b76aa271be9c0920(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:14:52,346 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:52,346 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/C, priority=13, startTime=1733814891658; duration=0sec 2024-12-10T07:14:52,346 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:52,346 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:C 2024-12-10T07:14:52,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741887_1063 (size=12151) 2024-12-10T07:14:52,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814952525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:52,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814952525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:52,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:52,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814952536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:52,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814952536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:52,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:52,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:52,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:52,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:52,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:52,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T07:14:52,754 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/299b8c6b31a74d21a61e95845a5453b5 2024-12-10T07:14:52,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/8a3c30f62d434009800efe8d86e583b6 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/8a3c30f62d434009800efe8d86e583b6 2024-12-10T07:14:52,786 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/8a3c30f62d434009800efe8d86e583b6, entries=150, sequenceid=236, filesize=11.9 K 2024-12-10T07:14:52,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/8bd2cf5c7c984a40b2b575e50489b515 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/8bd2cf5c7c984a40b2b575e50489b515 2024-12-10T07:14:52,798 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/8bd2cf5c7c984a40b2b575e50489b515, entries=150, sequenceid=236, filesize=11.9 K 2024-12-10T07:14:52,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/299b8c6b31a74d21a61e95845a5453b5 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/299b8c6b31a74d21a61e95845a5453b5 2024-12-10T07:14:52,807 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/299b8c6b31a74d21a61e95845a5453b5, entries=150, sequenceid=236, filesize=11.9 K 2024-12-10T07:14:52,813 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 4204f23293292645e34a5d93616d8bf1 in 1115ms, sequenceid=236, compaction requested=false 2024-12-10T07:14:52,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:52,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:52,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-10T07:14:52,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-10T07:14:52,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-10T07:14:52,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2820 sec 2024-12-10T07:14:52,826 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.2940 sec 2024-12-10T07:14:53,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:53,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4204f23293292645e34a5d93616d8bf1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T07:14:53,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=A 2024-12-10T07:14:53,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:53,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=B 2024-12-10T07:14:53,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:53,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=C 2024-12-10T07:14:53,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:53,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/0e59a1276841478bbe479869cd70c2b2 is 50, key is test_row_0/A:col10/1733814893064/Put/seqid=0 2024-12-10T07:14:53,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741888_1064 (size=12251) 2024-12-10T07:14:53,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:53,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814953141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:53,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:53,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814953156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:53,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814953161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:53,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:53,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:53,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:53,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814953169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:53,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:53,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:53,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814953256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:53,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:53,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:53,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814953275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:53,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:53,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814953284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:53,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:53,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814953287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:53,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:53,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:53,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:53,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:53,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814953481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:53,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:53,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:53,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814953505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:53,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:53,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:53,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814953505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:53,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:53,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:53,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814953505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:53,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:53,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/0e59a1276841478bbe479869cd70c2b2 2024-12-10T07:14:53,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/fb85be95ca0a4ef0ba66ad6728df6fb0 is 50, key is test_row_0/B:col10/1733814893064/Put/seqid=0 2024-12-10T07:14:53,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741889_1065 (size=12251) 2024-12-10T07:14:53,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/fb85be95ca0a4ef0ba66ad6728df6fb0 2024-12-10T07:14:53,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/fe8408da8c6f4dca8ca5d2d76fbae385 is 50, key is test_row_0/C:col10/1733814893064/Put/seqid=0 2024-12-10T07:14:53,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-10T07:14:53,689 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:14:53,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741890_1066 (size=12251) 2024-12-10T07:14:53,693 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:14:53,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/fe8408da8c6f4dca8ca5d2d76fbae385 2024-12-10T07:14:53,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-10T07:14:53,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T07:14:53,707 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:14:53,712 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:14:53,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:14:53,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/0e59a1276841478bbe479869cd70c2b2 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/0e59a1276841478bbe479869cd70c2b2 2024-12-10T07:14:53,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/0e59a1276841478bbe479869cd70c2b2, entries=150, sequenceid=258, filesize=12.0 K 2024-12-10T07:14:53,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/fb85be95ca0a4ef0ba66ad6728df6fb0 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/fb85be95ca0a4ef0ba66ad6728df6fb0 2024-12-10T07:14:53,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/fb85be95ca0a4ef0ba66ad6728df6fb0, entries=150, sequenceid=258, filesize=12.0 K 2024-12-10T07:14:53,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/fe8408da8c6f4dca8ca5d2d76fbae385 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/fe8408da8c6f4dca8ca5d2d76fbae385 2024-12-10T07:14:53,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814953796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:53,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:53,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/fe8408da8c6f4dca8ca5d2d76fbae385, entries=150, sequenceid=258, filesize=12.0 K 2024-12-10T07:14:53,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 4204f23293292645e34a5d93616d8bf1 in 741ms, sequenceid=258, compaction requested=true 2024-12-10T07:14:53,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:53,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:14:53,806 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:14:53,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:53,806 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:14:53,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:14:53,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:53,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:14:53,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:14:53,807 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:14:53,807 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:14:53,808 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/A is initiating minor compaction (all files) 2024-12-10T07:14:53,808 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/B is initiating minor compaction (all files) 2024-12-10T07:14:53,808 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/B in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:53,808 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/A in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:53,808 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/d4bb173a0c28475a927b8f4c6b6dcc0e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/8bd2cf5c7c984a40b2b575e50489b515, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/fb85be95ca0a4ef0ba66ad6728df6fb0] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=36.2 K 2024-12-10T07:14:53,808 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/1eb803766bf74c0da10f88eb02c2bafe, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/8a3c30f62d434009800efe8d86e583b6, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/0e59a1276841478bbe479869cd70c2b2] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=36.2 K 2024-12-10T07:14:53,809 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1eb803766bf74c0da10f88eb02c2bafe, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733814890617 2024-12-10T07:14:53,809 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting d4bb173a0c28475a927b8f4c6b6dcc0e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733814890617 2024-12-10T07:14:53,809 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bd2cf5c7c984a40b2b575e50489b515, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733814891439 2024-12-10T07:14:53,809 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a3c30f62d434009800efe8d86e583b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733814891439 2024-12-10T07:14:53,810 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting fb85be95ca0a4ef0ba66ad6728df6fb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733814891889 2024-12-10T07:14:53,810 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e59a1276841478bbe479869cd70c2b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733814891889 2024-12-10T07:14:53,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T07:14:53,823 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#B#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:14:53,824 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/b473f35cc0b84141aa071308ea3d1177 is 50, key is test_row_0/B:col10/1733814893064/Put/seqid=0 2024-12-10T07:14:53,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4204f23293292645e34a5d93616d8bf1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T07:14:53,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=A 2024-12-10T07:14:53,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:53,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=B 2024-12-10T07:14:53,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:53,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=C 2024-12-10T07:14:53,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:53,839 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#A#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:14:53,840 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/a632a0b0f50c4c7e87b38b4a179cf513 is 50, key is test_row_0/A:col10/1733814893064/Put/seqid=0 2024-12-10T07:14:53,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/6b4eccffc24c4a619b5f2b846df97ff2 is 50, key is test_row_0/A:col10/1733814893146/Put/seqid=0 2024-12-10T07:14:53,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T07:14:53,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:53,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:53,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:53,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:53,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:53,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:53,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741891_1067 (size=12865) 2024-12-10T07:14:53,901 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/b473f35cc0b84141aa071308ea3d1177 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/b473f35cc0b84141aa071308ea3d1177 2024-12-10T07:14:53,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741892_1068 (size=12865) 2024-12-10T07:14:53,916 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/B of 4204f23293292645e34a5d93616d8bf1 into b473f35cc0b84141aa071308ea3d1177(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:14:53,916 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:53,916 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/B, priority=13, startTime=1733814893806; duration=0sec 2024-12-10T07:14:53,917 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:14:53,917 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:B 2024-12-10T07:14:53,917 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:14:53,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814953920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:53,941 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:14:53,941 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/C is initiating minor compaction (all files) 2024-12-10T07:14:53,941 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/C in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:53,942 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/07929e3f1a1b45c5b76aa271be9c0920, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/299b8c6b31a74d21a61e95845a5453b5, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/fe8408da8c6f4dca8ca5d2d76fbae385] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=36.2 K 2024-12-10T07:14:53,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814953927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:53,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:53,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814953928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:53,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:53,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:53,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741893_1069 (size=14741) 2024-12-10T07:14:53,947 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 07929e3f1a1b45c5b76aa271be9c0920, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733814890617 2024-12-10T07:14:53,954 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 299b8c6b31a74d21a61e95845a5453b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733814891439 2024-12-10T07:14:53,954 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting fe8408da8c6f4dca8ca5d2d76fbae385, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733814891889 2024-12-10T07:14:53,995 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#C#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:14:53,997 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/bc4ea33f50094cbf9a21295c4a6ac6b4 is 50, key is test_row_0/C:col10/1733814893064/Put/seqid=0 2024-12-10T07:14:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T07:14:54,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741894_1070 (size=12865) 2024-12-10T07:14:54,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:54,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814954038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:54,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T07:14:54,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:54,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:54,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:54,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:54,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:54,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:54,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:54,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:54,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:54,049 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/bc4ea33f50094cbf9a21295c4a6ac6b4 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/bc4ea33f50094cbf9a21295c4a6ac6b4 2024-12-10T07:14:54,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:54,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814954049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:54,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:54,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:54,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:54,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814954061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:54,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:54,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:54,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:54,081 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/C of 4204f23293292645e34a5d93616d8bf1 into bc4ea33f50094cbf9a21295c4a6ac6b4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:14:54,081 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:54,081 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/C, priority=13, startTime=1733814893806; duration=0sec 2024-12-10T07:14:54,081 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:54,081 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:C 2024-12-10T07:14:54,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T07:14:54,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:54,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:54,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:54,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:54,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:54,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814954248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:54,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:54,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:54,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:54,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814954266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:54,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:54,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:54,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:54,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814954285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:54,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814954306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:54,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:54,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:54,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T07:14:54,338 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/a632a0b0f50c4c7e87b38b4a179cf513 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/a632a0b0f50c4c7e87b38b4a179cf513 2024-12-10T07:14:54,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/6b4eccffc24c4a619b5f2b846df97ff2 2024-12-10T07:14:54,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T07:14:54,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:54,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:54,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:54,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:54,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:54,363 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/A of 4204f23293292645e34a5d93616d8bf1 into a632a0b0f50c4c7e87b38b4a179cf513(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:14:54,363 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:54,363 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/A, priority=13, startTime=1733814893806; duration=0sec 2024-12-10T07:14:54,363 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:54,363 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:A 2024-12-10T07:14:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:54,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/e725a19575e3465eb2a98c506e558171 is 50, key is test_row_0/B:col10/1733814893146/Put/seqid=0 2024-12-10T07:14:54,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741895_1071 (size=12301) 2024-12-10T07:14:54,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T07:14:54,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:54,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:54,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:54,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:54,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:54,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814954554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:54,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:54,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:54,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:54,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814954577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:54,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814954594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:54,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T07:14:54,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:54,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:54,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:54,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:54,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T07:14:54,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T07:14:54,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:54,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:54,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:54,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:54,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:54,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/e725a19575e3465eb2a98c506e558171 2024-12-10T07:14:54,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/008b6be261e148f896616862beb71aa0 is 50, key is test_row_0/C:col10/1733814893146/Put/seqid=0 2024-12-10T07:14:55,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T07:14:55,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:55,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:55,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:55,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:55,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:55,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741896_1072 (size=12301) 2024-12-10T07:14:55,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/008b6be261e148f896616862beb71aa0 2024-12-10T07:14:55,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/6b4eccffc24c4a619b5f2b846df97ff2 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/6b4eccffc24c4a619b5f2b846df97ff2 2024-12-10T07:14:55,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/6b4eccffc24c4a619b5f2b846df97ff2, entries=200, sequenceid=273, filesize=14.4 K 2024-12-10T07:14:55,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814955064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:55,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/e725a19575e3465eb2a98c506e558171 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/e725a19575e3465eb2a98c506e558171 2024-12-10T07:14:55,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:55,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:55,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:55,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/e725a19575e3465eb2a98c506e558171, entries=150, sequenceid=273, filesize=12.0 K 2024-12-10T07:14:55,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/008b6be261e148f896616862beb71aa0 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/008b6be261e148f896616862beb71aa0 2024-12-10T07:14:55,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814955095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:55,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:55,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:55,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:55,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814955106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:55,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:55,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:55,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:55,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/008b6be261e148f896616862beb71aa0, entries=150, sequenceid=273, filesize=12.0 K 2024-12-10T07:14:55,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 4204f23293292645e34a5d93616d8bf1 in 1314ms, sequenceid=273, compaction requested=false 2024-12-10T07:14:55,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:55,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-10T07:14:55,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:55,182 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 4204f23293292645e34a5d93616d8bf1 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T07:14:55,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=A 2024-12-10T07:14:55,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:55,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=B 2024-12-10T07:14:55,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:55,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=C 2024-12-10T07:14:55,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:55,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/c2630c1735894aaf896b5587d5fa4592 is 50, key is test_row_0/A:col10/1733814893926/Put/seqid=0 2024-12-10T07:14:55,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741897_1073 (size=12301) 2024-12-10T07:14:55,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:55,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:55,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814955553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:55,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:55,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:55,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:55,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814955665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:55,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:55,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:55,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:55,705 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/c2630c1735894aaf896b5587d5fa4592 2024-12-10T07:14:55,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/1fbdfe6d172749bf931492f55c3bfcbe is 50, key is test_row_0/B:col10/1733814893926/Put/seqid=0 2024-12-10T07:14:55,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741898_1074 (size=12301) 2024-12-10T07:14:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T07:14:55,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814955877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:55,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:55,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:55,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:56,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814956075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:56,086 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 4204f23293292645e34a5d93616d8bf1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:14:56,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:56,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:56,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:56,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:56,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814956117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:56,122 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_2' in 4204f23293292645e34a5d93616d8bf1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:14:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:56,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:56,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814956125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:56,127 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 4204f23293292645e34a5d93616d8bf1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:14:56,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:56,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:56,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:56,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:56,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814956195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:56,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:56,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:56,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:56,259 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/1fbdfe6d172749bf931492f55c3bfcbe 2024-12-10T07:14:56,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/183c870f5b0141d1a3fc1668f3f9f324 is 50, key is test_row_0/C:col10/1733814893926/Put/seqid=0 2024-12-10T07:14:56,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741899_1075 (size=12301) 2024-12-10T07:14:56,323 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/183c870f5b0141d1a3fc1668f3f9f324 2024-12-10T07:14:56,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/c2630c1735894aaf896b5587d5fa4592 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/c2630c1735894aaf896b5587d5fa4592 2024-12-10T07:14:56,355 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/c2630c1735894aaf896b5587d5fa4592, entries=150, sequenceid=293, filesize=12.0 K 2024-12-10T07:14:56,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/1fbdfe6d172749bf931492f55c3bfcbe as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/1fbdfe6d172749bf931492f55c3bfcbe 2024-12-10T07:14:56,384 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/1fbdfe6d172749bf931492f55c3bfcbe, entries=150, sequenceid=293, filesize=12.0 K 2024-12-10T07:14:56,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/183c870f5b0141d1a3fc1668f3f9f324 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/183c870f5b0141d1a3fc1668f3f9f324 2024-12-10T07:14:56,405 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/183c870f5b0141d1a3fc1668f3f9f324, entries=150, sequenceid=293, filesize=12.0 K 2024-12-10T07:14:56,409 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4204f23293292645e34a5d93616d8bf1 in 1227ms, sequenceid=293, compaction requested=true 2024-12-10T07:14:56,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:56,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:56,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-10T07:14:56,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-10T07:14:56,417 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-10T07:14:56,417 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7010 sec 2024-12-10T07:14:56,425 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.7260 sec 2024-12-10T07:14:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:56,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4204f23293292645e34a5d93616d8bf1 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T07:14:56,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=A 2024-12-10T07:14:56,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:56,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=B 2024-12-10T07:14:56,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:56,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=C 2024-12-10T07:14:56,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:56,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/71585b455e0c403b9664ce7173fc6156 is 50, key is test_row_0/A:col10/1733814895543/Put/seqid=0 2024-12-10T07:14:56,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741900_1076 (size=12301) 2024-12-10T07:14:56,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/71585b455e0c403b9664ce7173fc6156 2024-12-10T07:14:56,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/885c94819d644246823de91c668eff6d is 50, key is test_row_0/B:col10/1733814895543/Put/seqid=0 2024-12-10T07:14:56,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741901_1077 (size=12301) 2024-12-10T07:14:56,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/885c94819d644246823de91c668eff6d 2024-12-10T07:14:56,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:56,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814956821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:56,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/79047079ebe64c2da1371a87f5ff3aa2 is 50, key is test_row_0/C:col10/1733814895543/Put/seqid=0 2024-12-10T07:14:56,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:56,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:56,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:56,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741902_1078 (size=12301) 2024-12-10T07:14:56,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:56,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814956936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:56,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:56,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:56,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:57,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814957156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:57,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:57,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:57,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:57,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/79047079ebe64c2da1371a87f5ff3aa2 2024-12-10T07:14:57,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/71585b455e0c403b9664ce7173fc6156 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/71585b455e0c403b9664ce7173fc6156 2024-12-10T07:14:57,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/71585b455e0c403b9664ce7173fc6156, entries=150, sequenceid=309, filesize=12.0 K 2024-12-10T07:14:57,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/885c94819d644246823de91c668eff6d as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/885c94819d644246823de91c668eff6d 2024-12-10T07:14:57,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/885c94819d644246823de91c668eff6d, entries=150, sequenceid=309, filesize=12.0 K 2024-12-10T07:14:57,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/79047079ebe64c2da1371a87f5ff3aa2 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/79047079ebe64c2da1371a87f5ff3aa2 2024-12-10T07:14:57,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/79047079ebe64c2da1371a87f5ff3aa2, entries=150, sequenceid=309, filesize=12.0 K 2024-12-10T07:14:57,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 4204f23293292645e34a5d93616d8bf1 in 660ms, sequenceid=309, compaction requested=true 2024-12-10T07:14:57,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:57,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:14:57,369 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:14:57,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:57,369 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:14:57,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:14:57,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:57,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:14:57,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:14:57,377 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52208 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:14:57,377 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49768 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:14:57,377 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/B is initiating minor compaction (all files) 2024-12-10T07:14:57,377 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/A is initiating minor compaction (all files) 2024-12-10T07:14:57,377 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/B in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:57,377 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/b473f35cc0b84141aa071308ea3d1177, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/e725a19575e3465eb2a98c506e558171, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/1fbdfe6d172749bf931492f55c3bfcbe, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/885c94819d644246823de91c668eff6d] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=48.6 K 2024-12-10T07:14:57,378 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/A in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:57,378 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting b473f35cc0b84141aa071308ea3d1177, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733814891889 2024-12-10T07:14:57,378 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/a632a0b0f50c4c7e87b38b4a179cf513, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/6b4eccffc24c4a619b5f2b846df97ff2, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/c2630c1735894aaf896b5587d5fa4592, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/71585b455e0c403b9664ce7173fc6156] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=51.0 K 2024-12-10T07:14:57,378 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting e725a19575e3465eb2a98c506e558171, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733814893135 2024-12-10T07:14:57,381 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting a632a0b0f50c4c7e87b38b4a179cf513, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733814891889 2024-12-10T07:14:57,381 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fbdfe6d172749bf931492f55c3bfcbe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733814893887 2024-12-10T07:14:57,382 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 885c94819d644246823de91c668eff6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733814895522 2024-12-10T07:14:57,384 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b4eccffc24c4a619b5f2b846df97ff2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733814893135 2024-12-10T07:14:57,394 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2630c1735894aaf896b5587d5fa4592, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733814893887 2024-12-10T07:14:57,398 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71585b455e0c403b9664ce7173fc6156, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733814895522 2024-12-10T07:14:57,423 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#B#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:14:57,424 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/9eb1b05b456c4860832231b04ac0ae59 is 50, key is test_row_0/B:col10/1733814895543/Put/seqid=0 2024-12-10T07:14:57,429 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#A#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:14:57,430 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/3f6fbc5356354985b4040a42d3cbd731 is 50, key is test_row_0/A:col10/1733814895543/Put/seqid=0 2024-12-10T07:14:57,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741903_1079 (size=13051) 2024-12-10T07:14:57,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741904_1080 (size=13051) 2024-12-10T07:14:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:57,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4204f23293292645e34a5d93616d8bf1 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T07:14:57,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=A 2024-12-10T07:14:57,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:57,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=B 2024-12-10T07:14:57,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:57,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=C 2024-12-10T07:14:57,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:57,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/0c7f0aec4aa5469193f0f4c2e6ebc54c is 50, key is test_row_0/A:col10/1733814896760/Put/seqid=0 2024-12-10T07:14:57,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741905_1081 (size=14741) 2024-12-10T07:14:57,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/0c7f0aec4aa5469193f0f4c2e6ebc54c 2024-12-10T07:14:57,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/59cb5f0e8ce84d85902eaeb9dded9775 is 50, key is test_row_0/B:col10/1733814896760/Put/seqid=0 2024-12-10T07:14:57,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741906_1082 (size=12301) 2024-12-10T07:14:57,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/59cb5f0e8ce84d85902eaeb9dded9775 2024-12-10T07:14:57,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:57,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/4c736fcdab31489f9c20aa0da5cc2e1d is 50, key is test_row_0/C:col10/1733814896760/Put/seqid=0 2024-12-10T07:14:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814957586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:57,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:57,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:57,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:57,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741907_1083 (size=12301) 2024-12-10T07:14:57,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/4c736fcdab31489f9c20aa0da5cc2e1d 2024-12-10T07:14:57,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/0c7f0aec4aa5469193f0f4c2e6ebc54c as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/0c7f0aec4aa5469193f0f4c2e6ebc54c 2024-12-10T07:14:57,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/0c7f0aec4aa5469193f0f4c2e6ebc54c, entries=200, sequenceid=326, filesize=14.4 K 2024-12-10T07:14:57,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/59cb5f0e8ce84d85902eaeb9dded9775 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/59cb5f0e8ce84d85902eaeb9dded9775 2024-12-10T07:14:57,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/59cb5f0e8ce84d85902eaeb9dded9775, entries=150, sequenceid=326, filesize=12.0 K 2024-12-10T07:14:57,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/4c736fcdab31489f9c20aa0da5cc2e1d as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/4c736fcdab31489f9c20aa0da5cc2e1d 2024-12-10T07:14:57,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:57,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814957705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:57,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:57,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:57,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/4c736fcdab31489f9c20aa0da5cc2e1d, entries=150, sequenceid=326, filesize=12.0 K 2024-12-10T07:14:57,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4204f23293292645e34a5d93616d8bf1 in 227ms, sequenceid=326, compaction requested=true 2024-12-10T07:14:57,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:57,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:A, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:14:57,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T07:14:57,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:B, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:14:57,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-10T07:14:57,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:14:57,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-10T07:14:57,872 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/9eb1b05b456c4860832231b04ac0ae59 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/9eb1b05b456c4860832231b04ac0ae59 2024-12-10T07:14:57,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-10T07:14:57,886 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:14:57,886 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:14:57,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-10T07:14:57,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T07:14:57,890 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:14:57,892 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:14:57,892 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:14:57,901 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/B of 4204f23293292645e34a5d93616d8bf1 into 9eb1b05b456c4860832231b04ac0ae59(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:14:57,903 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:57,903 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/B, priority=12, startTime=1733814897369; duration=0sec 2024-12-10T07:14:57,903 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-10T07:14:57,903 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:B 2024-12-10T07:14:57,903 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-12-10T07:14:57,905 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-10T07:14:57,905 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-10T07:14:57,906 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. because compaction request was cancelled 2024-12-10T07:14:57,907 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:A 2024-12-10T07:14:57,907 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-10T07:14:57,910 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62069 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-10T07:14:57,910 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/C is initiating minor compaction (all files) 2024-12-10T07:14:57,910 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/C in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:57,910 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/bc4ea33f50094cbf9a21295c4a6ac6b4, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/008b6be261e148f896616862beb71aa0, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/183c870f5b0141d1a3fc1668f3f9f324, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/79047079ebe64c2da1371a87f5ff3aa2, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/4c736fcdab31489f9c20aa0da5cc2e1d] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=60.6 K 2024-12-10T07:14:57,911 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting bc4ea33f50094cbf9a21295c4a6ac6b4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733814891889 2024-12-10T07:14:57,911 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 008b6be261e148f896616862beb71aa0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733814893135 2024-12-10T07:14:57,912 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 183c870f5b0141d1a3fc1668f3f9f324, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733814893887 2024-12-10T07:14:57,912 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 79047079ebe64c2da1371a87f5ff3aa2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733814895522 2024-12-10T07:14:57,913 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c736fcdab31489f9c20aa0da5cc2e1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733814896760 2024-12-10T07:14:57,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:57,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4204f23293292645e34a5d93616d8bf1 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T07:14:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=A 2024-12-10T07:14:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=B 2024-12-10T07:14:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=C 2024-12-10T07:14:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:57,929 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/3f6fbc5356354985b4040a42d3cbd731 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/3f6fbc5356354985b4040a42d3cbd731 2024-12-10T07:14:57,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/f44bf120eb4a49208b98a096b8624425 is 50, key is test_row_0/A:col10/1733814897918/Put/seqid=0 2024-12-10T07:14:57,946 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/A of 4204f23293292645e34a5d93616d8bf1 into 3f6fbc5356354985b4040a42d3cbd731(size=12.7 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:14:57,946 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:57,946 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/A, priority=12, startTime=1733814897368; duration=0sec 2024-12-10T07:14:57,946 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T07:14:57,946 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:A 2024-12-10T07:14:57,946 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:B 2024-12-10T07:14:57,947 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-12-10T07:14:57,947 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-10T07:14:57,947 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-10T07:14:57,947 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. because compaction request was cancelled 2024-12-10T07:14:57,947 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:C 2024-12-10T07:14:57,947 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-10T07:14:57,948 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#C#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:14:57,948 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/18649dc3bae44c52aaaf2e2b784972fe is 50, key is test_row_0/C:col10/1733814896760/Put/seqid=0 2024-12-10T07:14:57,952 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-10T07:14:57,952 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-10T07:14:57,952 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. because compaction request was cancelled 2024-12-10T07:14:57,952 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:B 2024-12-10T07:14:57,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741908_1084 (size=12301) 2024-12-10T07:14:57,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/f44bf120eb4a49208b98a096b8624425 2024-12-10T07:14:57,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T07:14:58,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741909_1085 (size=13085) 2024-12-10T07:14:58,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/b4f669eae1bd49eeb54dc2841f6b7b77 is 50, key is test_row_0/B:col10/1733814897918/Put/seqid=0 2024-12-10T07:14:58,021 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/18649dc3bae44c52aaaf2e2b784972fe as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/18649dc3bae44c52aaaf2e2b784972fe 2024-12-10T07:14:58,033 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/C of 4204f23293292645e34a5d93616d8bf1 into 18649dc3bae44c52aaaf2e2b784972fe(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:14:58,033 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:58,033 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/C, priority=11, startTime=1733814897714; duration=0sec 2024-12-10T07:14:58,034 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:58,034 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:C 2024-12-10T07:14:58,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741910_1086 (size=12301) 2024-12-10T07:14:58,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/b4f669eae1bd49eeb54dc2841f6b7b77 2024-12-10T07:14:58,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-10T07:14:58,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:58,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:58,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:58,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:58,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:58,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:58,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/0175dc1ed9674896982ca384359bf784 is 50, key is test_row_0/C:col10/1733814897918/Put/seqid=0 2024-12-10T07:14:58,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:58,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814958074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:58,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:58,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:58,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:58,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:58,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59146 deadline: 1733814958105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:58,116 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 4204f23293292645e34a5d93616d8bf1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:14:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:58,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741911_1087 (size=12301) 2024-12-10T07:14:58,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/0175dc1ed9674896982ca384359bf784 2024-12-10T07:14:58,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:58,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59160 deadline: 1733814958135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:58,142 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_2' in 4204f23293292645e34a5d93616d8bf1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:14:58,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:58,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:58,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:58,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:58,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59136 deadline: 1733814958135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:58,144 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 4204f23293292645e34a5d93616d8bf1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:14:58,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:58,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:58,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:58,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/f44bf120eb4a49208b98a096b8624425 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/f44bf120eb4a49208b98a096b8624425 2024-12-10T07:14:58,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/f44bf120eb4a49208b98a096b8624425, entries=150, sequenceid=343, filesize=12.0 K 2024-12-10T07:14:58,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:58,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814958186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:58,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/b4f669eae1bd49eeb54dc2841f6b7b77 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/b4f669eae1bd49eeb54dc2841f6b7b77 2024-12-10T07:14:58,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T07:14:58,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-10T07:14:58,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:58,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:58,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:58,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:58,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:58,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:58,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/b4f669eae1bd49eeb54dc2841f6b7b77, entries=150, sequenceid=343, filesize=12.0 K 2024-12-10T07:14:58,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/0175dc1ed9674896982ca384359bf784 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/0175dc1ed9674896982ca384359bf784 2024-12-10T07:14:58,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/0175dc1ed9674896982ca384359bf784, entries=150, sequenceid=343, filesize=12.0 K 2024-12-10T07:14:58,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 4204f23293292645e34a5d93616d8bf1 in 376ms, sequenceid=343, compaction requested=true 2024-12-10T07:14:58,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:58,300 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:14:58,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:14:58,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:58,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:14:58,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:58,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:14:58,300 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:14:58,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:14:58,309 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:14:58,309 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/B is initiating minor compaction (all files) 2024-12-10T07:14:58,309 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/B in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:58,309 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/9eb1b05b456c4860832231b04ac0ae59, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/59cb5f0e8ce84d85902eaeb9dded9775, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/b4f669eae1bd49eeb54dc2841f6b7b77] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=36.8 K 2024-12-10T07:14:58,310 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:14:58,310 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/A is initiating minor compaction (all files) 2024-12-10T07:14:58,310 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/A in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:58,310 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/3f6fbc5356354985b4040a42d3cbd731, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/0c7f0aec4aa5469193f0f4c2e6ebc54c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/f44bf120eb4a49208b98a096b8624425] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=39.2 K 2024-12-10T07:14:58,313 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f6fbc5356354985b4040a42d3cbd731, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733814895522 2024-12-10T07:14:58,313 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 9eb1b05b456c4860832231b04ac0ae59, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733814895522 2024-12-10T07:14:58,314 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c7f0aec4aa5469193f0f4c2e6ebc54c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733814896760 2024-12-10T07:14:58,314 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 59cb5f0e8ce84d85902eaeb9dded9775, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733814896760 2024-12-10T07:14:58,315 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting f44bf120eb4a49208b98a096b8624425, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733814897580 2024-12-10T07:14:58,318 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting b4f669eae1bd49eeb54dc2841f6b7b77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733814897580 2024-12-10T07:14:58,358 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#A#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:14:58,358 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/8a3b14e702ae4136a6d06c0924566efc is 50, key is test_row_0/A:col10/1733814897918/Put/seqid=0 2024-12-10T07:14:58,370 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#B#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:14:58,370 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/19d61cae206148c586849302981b2b04 is 50, key is test_row_0/B:col10/1733814897918/Put/seqid=0 2024-12-10T07:14:58,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-10T07:14:58,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:58,375 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 4204f23293292645e34a5d93616d8bf1 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T07:14:58,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=A 2024-12-10T07:14:58,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:58,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=B 2024-12-10T07:14:58,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:58,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=C 2024-12-10T07:14:58,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:58,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/5f79f3f5939e4db3820bf8cd7a8981d3 is 50, key is test_row_0/A:col10/1733814898048/Put/seqid=0 2024-12-10T07:14:58,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:58,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:58,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741913_1089 (size=13153) 2024-12-10T07:14:58,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741912_1088 (size=13153) 2024-12-10T07:14:58,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741914_1090 (size=12301) 2024-12-10T07:14:58,486 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/19d61cae206148c586849302981b2b04 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/19d61cae206148c586849302981b2b04 2024-12-10T07:14:58,492 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/5f79f3f5939e4db3820bf8cd7a8981d3 2024-12-10T07:14:58,513 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/8a3b14e702ae4136a6d06c0924566efc as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/8a3b14e702ae4136a6d06c0924566efc 2024-12-10T07:14:58,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T07:14:58,525 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/B of 4204f23293292645e34a5d93616d8bf1 into 19d61cae206148c586849302981b2b04(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:14:58,525 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:58,525 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/B, priority=13, startTime=1733814898300; duration=0sec 2024-12-10T07:14:58,526 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:14:58,526 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:B 2024-12-10T07:14:58,526 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-10T07:14:58,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/d8d683600e6c43cdabb46733e10a282e is 50, key is test_row_0/B:col10/1733814898048/Put/seqid=0 2024-12-10T07:14:58,531 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-10T07:14:58,531 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-10T07:14:58,531 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. because compaction request was cancelled 2024-12-10T07:14:58,531 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:C 2024-12-10T07:14:58,535 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/A of 4204f23293292645e34a5d93616d8bf1 into 8a3b14e702ae4136a6d06c0924566efc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:14:58,535 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:58,535 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/A, priority=13, startTime=1733814898300; duration=0sec 2024-12-10T07:14:58,536 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:58,536 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:A 2024-12-10T07:14:58,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:58,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814958558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:58,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:58,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:58,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:58,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741915_1091 (size=12301) 2024-12-10T07:14:58,569 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/d8d683600e6c43cdabb46733e10a282e 2024-12-10T07:14:58,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/863631ef5d9d4c218c9c2cfb0c893129 is 50, key is test_row_0/C:col10/1733814898048/Put/seqid=0 2024-12-10T07:14:58,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741916_1092 (size=12301) 2024-12-10T07:14:58,615 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/863631ef5d9d4c218c9c2cfb0c893129 2024-12-10T07:14:58,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/5f79f3f5939e4db3820bf8cd7a8981d3 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/5f79f3f5939e4db3820bf8cd7a8981d3 2024-12-10T07:14:58,638 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/5f79f3f5939e4db3820bf8cd7a8981d3, entries=150, sequenceid=361, filesize=12.0 K 2024-12-10T07:14:58,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/d8d683600e6c43cdabb46733e10a282e as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/d8d683600e6c43cdabb46733e10a282e 2024-12-10T07:14:58,654 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/d8d683600e6c43cdabb46733e10a282e, entries=150, sequenceid=361, filesize=12.0 K 2024-12-10T07:14:58,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/863631ef5d9d4c218c9c2cfb0c893129 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/863631ef5d9d4c218c9c2cfb0c893129 2024-12-10T07:14:58,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814958665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:58,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:58,681 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/863631ef5d9d4c218c9c2cfb0c893129, entries=150, sequenceid=361, filesize=12.0 K 2024-12-10T07:14:58,683 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 4204f23293292645e34a5d93616d8bf1 in 308ms, sequenceid=361, compaction requested=true 2024-12-10T07:14:58,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:58,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:58,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-10T07:14:58,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-10T07:14:58,689 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-10T07:14:58,689 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 793 msec 2024-12-10T07:14:58,695 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 807 msec 2024-12-10T07:14:58,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4204f23293292645e34a5d93616d8bf1 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T07:14:58,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=A 2024-12-10T07:14:58,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:58,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=B 2024-12-10T07:14:58,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:58,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=C 2024-12-10T07:14:58,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:14:58,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:14:58,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/bd75bdbc39c243979ba803a812141de1 is 50, key is test_row_0/A:col10/1733814898554/Put/seqid=0 2024-12-10T07:14:58,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741917_1093 (size=14741) 2024-12-10T07:14:59,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:59,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814959017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:59,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:59,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-10T07:14:59,037 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:14:59,038 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:14:59,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-10T07:14:59,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-10T07:14:59,057 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:14:59,061 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:14:59,062 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:14:59,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:59,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814959125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:59,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:59,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:59,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:59,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-10T07:14:59,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T07:14:59,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:59,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:59,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:59,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:59,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:59,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/bd75bdbc39c243979ba803a812141de1 2024-12-10T07:14:59,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:59,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814959335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:59,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:59,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:59,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:59,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/6de1ea3183be4e1eacdd907fa37f88bb is 50, key is test_row_0/B:col10/1733814898554/Put/seqid=0 2024-12-10T07:14:59,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-10T07:14:59,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741918_1094 (size=12301) 2024-12-10T07:14:59,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T07:14:59,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:59,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:59,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:59,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:59,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:59,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:59,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T07:14:59,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:59,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:59,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:59,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:59,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:59,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:59,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:14:59,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814959651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:14:59,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:59,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:14:59,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:14:59,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-10T07:14:59,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T07:14:59,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:59,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:59,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:59,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:59,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:59,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:59,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/6de1ea3183be4e1eacdd907fa37f88bb 2024-12-10T07:14:59,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/238e32d427f6466e873ff8b1bb1da0e9 is 50, key is test_row_0/C:col10/1733814898554/Put/seqid=0 2024-12-10T07:14:59,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741919_1095 (size=12301) 2024-12-10T07:14:59,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/238e32d427f6466e873ff8b1bb1da0e9 2024-12-10T07:14:59,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/bd75bdbc39c243979ba803a812141de1 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/bd75bdbc39c243979ba803a812141de1 2024-12-10T07:14:59,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/bd75bdbc39c243979ba803a812141de1, entries=200, sequenceid=380, filesize=14.4 K 2024-12-10T07:14:59,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/6de1ea3183be4e1eacdd907fa37f88bb as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/6de1ea3183be4e1eacdd907fa37f88bb 2024-12-10T07:14:59,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/6de1ea3183be4e1eacdd907fa37f88bb, entries=150, sequenceid=380, filesize=12.0 K 2024-12-10T07:14:59,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T07:14:59,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:59,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:14:59,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:59,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:59,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:59,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/238e32d427f6466e873ff8b1bb1da0e9 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/238e32d427f6466e873ff8b1bb1da0e9 2024-12-10T07:14:59,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:14:59,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/238e32d427f6466e873ff8b1bb1da0e9, entries=150, sequenceid=380, filesize=12.0 K 2024-12-10T07:14:59,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4204f23293292645e34a5d93616d8bf1 in 1042ms, sequenceid=380, compaction requested=true 2024-12-10T07:14:59,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:14:59,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:14:59,934 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:14:59,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:14:59,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:14:59,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:14:59,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:14:59,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T07:14:59,934 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:14:59,947 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:14:59,947 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/A is initiating minor compaction (all files) 2024-12-10T07:14:59,947 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/A in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:59,947 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/8a3b14e702ae4136a6d06c0924566efc, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/5f79f3f5939e4db3820bf8cd7a8981d3, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/bd75bdbc39c243979ba803a812141de1] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=39.3 K 2024-12-10T07:14:59,948 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:14:59,948 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a3b14e702ae4136a6d06c0924566efc, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733814897580 2024-12-10T07:14:59,948 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/C is initiating minor compaction (all files) 2024-12-10T07:14:59,948 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/C in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:14:59,948 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/18649dc3bae44c52aaaf2e2b784972fe, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/0175dc1ed9674896982ca384359bf784, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/863631ef5d9d4c218c9c2cfb0c893129, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/238e32d427f6466e873ff8b1bb1da0e9] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=48.8 K 2024-12-10T07:14:59,948 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f79f3f5939e4db3820bf8cd7a8981d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733814898032 2024-12-10T07:14:59,948 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18649dc3bae44c52aaaf2e2b784972fe, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733814896760 2024-12-10T07:14:59,949 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting bd75bdbc39c243979ba803a812141de1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733814898517 2024-12-10T07:14:59,953 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0175dc1ed9674896982ca384359bf784, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733814897580 2024-12-10T07:14:59,954 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 863631ef5d9d4c218c9c2cfb0c893129, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733814898032 2024-12-10T07:14:59,955 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 238e32d427f6466e873ff8b1bb1da0e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733814898547 2024-12-10T07:14:59,959 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#A#compaction#83 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:14:59,960 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/a8e6a28556044cc093b86288d6d84552 is 50, key is test_row_0/A:col10/1733814898554/Put/seqid=0 2024-12-10T07:14:59,992 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#C#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:14:59,993 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/219329cfa2084c8986a5f7d0b828da95 is 50, key is test_row_0/C:col10/1733814898554/Put/seqid=0 2024-12-10T07:15:00,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741920_1096 (size=13255) 2024-12-10T07:15:00,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-10T07:15:00,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:15:00,047 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 4204f23293292645e34a5d93616d8bf1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T07:15:00,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=A 2024-12-10T07:15:00,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:00,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=B 2024-12-10T07:15:00,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:00,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=C 2024-12-10T07:15:00,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:00,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741921_1097 (size=13221) 2024-12-10T07:15:00,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/fcb0f2f688d040a9b6c7ecbb0ab7c82b is 50, key is test_row_0/A:col10/1733814899015/Put/seqid=0 2024-12-10T07:15:00,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741922_1098 (size=12301) 2024-12-10T07:15:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:15:00,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:15:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-10T07:15:00,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:00,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814960311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:00,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:00,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:00,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:00,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:00,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59110 deadline: 1733814960413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:00,429 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_2' in 4204f23293292645e34a5d93616d8bf1 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18341 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:15:00,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:00,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:00,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:00,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:00,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814960425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:00,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:00,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:00,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:00,489 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/a8e6a28556044cc093b86288d6d84552 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/a8e6a28556044cc093b86288d6d84552 2024-12-10T07:15:00,494 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/219329cfa2084c8986a5f7d0b828da95 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/219329cfa2084c8986a5f7d0b828da95 2024-12-10T07:15:00,503 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/C of 4204f23293292645e34a5d93616d8bf1 into 219329cfa2084c8986a5f7d0b828da95(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:00,503 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:15:00,503 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/C, priority=12, startTime=1733814899934; duration=0sec 2024-12-10T07:15:00,503 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:00,503 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:C 2024-12-10T07:15:00,503 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:00,512 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:00,512 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/B is initiating minor compaction (all files) 2024-12-10T07:15:00,512 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/B in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:15:00,512 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/19d61cae206148c586849302981b2b04, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/d8d683600e6c43cdabb46733e10a282e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/6de1ea3183be4e1eacdd907fa37f88bb] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=36.9 K 2024-12-10T07:15:00,512 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/fcb0f2f688d040a9b6c7ecbb0ab7c82b 2024-12-10T07:15:00,513 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19d61cae206148c586849302981b2b04, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733814897580 2024-12-10T07:15:00,513 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8d683600e6c43cdabb46733e10a282e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733814898032 2024-12-10T07:15:00,514 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6de1ea3183be4e1eacdd907fa37f88bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733814898547 2024-12-10T07:15:00,526 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/A of 4204f23293292645e34a5d93616d8bf1 into a8e6a28556044cc093b86288d6d84552(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:00,526 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:15:00,526 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/A, priority=13, startTime=1733814899934; duration=0sec 2024-12-10T07:15:00,526 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:00,526 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:A 2024-12-10T07:15:00,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/df907149234c4fe7990ab25683887373 is 50, key is test_row_0/B:col10/1733814899015/Put/seqid=0 2024-12-10T07:15:00,539 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#B#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:00,540 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/7282959d44f74700b138d535ccc7e034 is 50, key is test_row_0/B:col10/1733814898554/Put/seqid=0 2024-12-10T07:15:00,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741923_1099 (size=12301) 2024-12-10T07:15:00,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741924_1100 (size=13255) 2024-12-10T07:15:00,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:00,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814960644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:00,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:00,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:00,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:00,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:00,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814960955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:00,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:00,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:00,988 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/df907149234c4fe7990ab25683887373 2024-12-10T07:15:01,021 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/7282959d44f74700b138d535ccc7e034 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/7282959d44f74700b138d535ccc7e034 2024-12-10T07:15:01,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/1eacff445a2a4b46a4a6f4a896dc03ce is 50, key is test_row_0/C:col10/1733814899015/Put/seqid=0 2024-12-10T07:15:01,058 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/B of 4204f23293292645e34a5d93616d8bf1 into 7282959d44f74700b138d535ccc7e034(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:01,058 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:15:01,058 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/B, priority=13, startTime=1733814899934; duration=0sec 2024-12-10T07:15:01,058 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:01,058 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:B 2024-12-10T07:15:01,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741925_1101 (size=12301) 2024-12-10T07:15:01,094 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/1eacff445a2a4b46a4a6f4a896dc03ce 2024-12-10T07:15:01,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/fcb0f2f688d040a9b6c7ecbb0ab7c82b as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/fcb0f2f688d040a9b6c7ecbb0ab7c82b 2024-12-10T07:15:01,112 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/fcb0f2f688d040a9b6c7ecbb0ab7c82b, entries=150, sequenceid=395, filesize=12.0 K 2024-12-10T07:15:01,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/df907149234c4fe7990ab25683887373 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/df907149234c4fe7990ab25683887373 2024-12-10T07:15:01,122 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/df907149234c4fe7990ab25683887373, entries=150, sequenceid=395, filesize=12.0 K 2024-12-10T07:15:01,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/1eacff445a2a4b46a4a6f4a896dc03ce as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/1eacff445a2a4b46a4a6f4a896dc03ce 2024-12-10T07:15:01,134 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/1eacff445a2a4b46a4a6f4a896dc03ce, entries=150, sequenceid=395, filesize=12.0 K 2024-12-10T07:15:01,135 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 4204f23293292645e34a5d93616d8bf1 in 1088ms, sequenceid=395, compaction requested=false 2024-12-10T07:15:01,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:15:01,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:15:01,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-10T07:15:01,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-10T07:15:01,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-10T07:15:01,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0840 sec 2024-12-10T07:15:01,159 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.1180 sec 2024-12-10T07:15:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-10T07:15:01,195 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:15:01,195 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:15:01,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-10T07:15:01,200 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:15:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-10T07:15:01,204 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:15:01,204 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:15:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-10T07:15:01,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-10T07:15:01,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:15:01,362 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 4204f23293292645e34a5d93616d8bf1 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T07:15:01,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=A 2024-12-10T07:15:01,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:01,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=B 2024-12-10T07:15:01,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:01,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=C 2024-12-10T07:15:01,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:01,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/26759537b28749b4b433db1ca5d2028c is 50, key is test_row_0/A:col10/1733814900295/Put/seqid=0 2024-12-10T07:15:01,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741926_1102 (size=12301) 2024-12-10T07:15:01,401 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/26759537b28749b4b433db1ca5d2028c 2024-12-10T07:15:01,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/14d907db65134a1cb3c55e2cbfbc133c is 50, key is test_row_0/B:col10/1733814900295/Put/seqid=0 2024-12-10T07:15:01,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741927_1103 (size=12301) 2024-12-10T07:15:01,461 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/14d907db65134a1cb3c55e2cbfbc133c 2024-12-10T07:15:01,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. as already flushing 2024-12-10T07:15:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:15:01,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/86e18e4056954f6bb0bd233b39caedb2 is 50, key is test_row_0/C:col10/1733814900295/Put/seqid=0 2024-12-10T07:15:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-10T07:15:01,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741928_1104 (size=12301) 2024-12-10T07:15:01,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814961574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:01,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:01,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:01,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:01,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814961685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:01,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:01,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:01,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:01,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-10T07:15:01,885 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-10T07:15:01,885 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:15:01,886 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-10T07:15:01,886 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:15:01,888 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-10T07:15:01,888 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:15:01,890 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-10T07:15:01,890 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:15:01,891 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:01,891 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:01,891 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:01,892 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:01,892 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:01,892 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:15:01,892 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:01,892 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:15:01,892 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:01,893 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:15:01,893 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:01,893 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:15:01,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59162 deadline: 1733814961895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:01,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:01,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:01,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:01,929 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/86e18e4056954f6bb0bd233b39caedb2 2024-12-10T07:15:01,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/26759537b28749b4b433db1ca5d2028c as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/26759537b28749b4b433db1ca5d2028c 2024-12-10T07:15:01,943 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/26759537b28749b4b433db1ca5d2028c, entries=150, sequenceid=415, filesize=12.0 K 2024-12-10T07:15:01,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/14d907db65134a1cb3c55e2cbfbc133c as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/14d907db65134a1cb3c55e2cbfbc133c 2024-12-10T07:15:01,955 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/14d907db65134a1cb3c55e2cbfbc133c, entries=150, sequenceid=415, filesize=12.0 K 2024-12-10T07:15:01,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/86e18e4056954f6bb0bd233b39caedb2 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/86e18e4056954f6bb0bd233b39caedb2 2024-12-10T07:15:01,962 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/86e18e4056954f6bb0bd233b39caedb2, entries=150, sequenceid=415, filesize=12.0 K 2024-12-10T07:15:01,963 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4204f23293292645e34a5d93616d8bf1 in 601ms, sequenceid=415, compaction requested=true 2024-12-10T07:15:01,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:15:01,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:15:01,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-10T07:15:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-10T07:15:01,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-10T07:15:01,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 761 msec 2024-12-10T07:15:01,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 775 msec 2024-12-10T07:15:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:15:02,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4204f23293292645e34a5d93616d8bf1 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T07:15:02,147 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-10T07:15:02,147 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:15:02,147 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:02,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=A 2024-12-10T07:15:02,147 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:02,147 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:15:02,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:02,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=B 2024-12-10T07:15:02,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:02,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=C 2024-12-10T07:15:02,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:02,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/52af774af51740deb684b099db1c0582 is 50, key is test_row_0/A:col10/1733814901568/Put/seqid=0 2024-12-10T07:15:02,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741929_1105 (size=12301) 2024-12-10T07:15:02,157 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-10T07:15:02,157 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:15:02,157 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:02,157 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:02,157 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:15:02,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/52af774af51740deb684b099db1c0582 2024-12-10T07:15:02,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/3ddae56bcca24313ac09a02ddbdfb0b4 is 50, key is test_row_0/B:col10/1733814901568/Put/seqid=0 2024-12-10T07:15:02,169 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-10T07:15:02,169 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:15:02,169 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:02,169 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:02,169 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:15:02,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741930_1106 (size=12301) 2024-12-10T07:15:02,206 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-10T07:15:02,206 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:15:02,206 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:02,206 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:02,206 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:15:02,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-10T07:15:02,335 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:15:02,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/3ddae56bcca24313ac09a02ddbdfb0b4 2024-12-10T07:15:02,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/fa4cdefdbe7e49b184e8411caf659066 is 50, key is test_row_0/C:col10/1733814901568/Put/seqid=0 2024-12-10T07:15:02,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741931_1107 (size=12301) 2024-12-10T07:15:02,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/fa4cdefdbe7e49b184e8411caf659066 2024-12-10T07:15:02,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/52af774af51740deb684b099db1c0582 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/52af774af51740deb684b099db1c0582 2024-12-10T07:15:02,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/52af774af51740deb684b099db1c0582, entries=150, sequenceid=431, filesize=12.0 K 2024-12-10T07:15:02,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/3ddae56bcca24313ac09a02ddbdfb0b4 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/3ddae56bcca24313ac09a02ddbdfb0b4 2024-12-10T07:15:03,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/3ddae56bcca24313ac09a02ddbdfb0b4, entries=150, sequenceid=431, filesize=12.0 K 2024-12-10T07:15:03,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/fa4cdefdbe7e49b184e8411caf659066 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/fa4cdefdbe7e49b184e8411caf659066 2024-12-10T07:15:03,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/fa4cdefdbe7e49b184e8411caf659066, entries=150, sequenceid=431, filesize=12.0 K 2024-12-10T07:15:03,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=20.13 KB/20610 for 4204f23293292645e34a5d93616d8bf1 in 862ms, sequenceid=431, compaction requested=true 2024-12-10T07:15:03,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:15:03,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:15:03,009 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:15:03,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:03,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:15:03,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:03,009 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:15:03,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4204f23293292645e34a5d93616d8bf1:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:15:03,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:03,010 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:15:03,010 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:15:03,010 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/A is initiating minor compaction (all files) 2024-12-10T07:15:03,010 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/A in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:15:03,010 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/B is initiating minor compaction (all files) 2024-12-10T07:15:03,010 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/B in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:15:03,010 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/a8e6a28556044cc093b86288d6d84552, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/fcb0f2f688d040a9b6c7ecbb0ab7c82b, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/26759537b28749b4b433db1ca5d2028c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/52af774af51740deb684b099db1c0582] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=49.0 K 2024-12-10T07:15:03,010 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/7282959d44f74700b138d535ccc7e034, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/df907149234c4fe7990ab25683887373, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/14d907db65134a1cb3c55e2cbfbc133c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/3ddae56bcca24313ac09a02ddbdfb0b4] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=49.0 K 2024-12-10T07:15:03,010 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8e6a28556044cc093b86288d6d84552, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733814898547 2024-12-10T07:15:03,011 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 7282959d44f74700b138d535ccc7e034, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733814898547 2024-12-10T07:15:03,011 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcb0f2f688d040a9b6c7ecbb0ab7c82b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733814898964 2024-12-10T07:15:03,011 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting df907149234c4fe7990ab25683887373, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733814898964 2024-12-10T07:15:03,011 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26759537b28749b4b433db1ca5d2028c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733814900284 2024-12-10T07:15:03,011 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 14d907db65134a1cb3c55e2cbfbc133c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733814900284 2024-12-10T07:15:03,011 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52af774af51740deb684b099db1c0582, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733814901544 2024-12-10T07:15:03,012 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ddae56bcca24313ac09a02ddbdfb0b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733814901544 2024-12-10T07:15:03,023 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#A#compaction#96 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:03,024 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/d62b2eb7a22c4f41b0dec5900bd3233b is 50, key is test_row_0/A:col10/1733814901568/Put/seqid=0 2024-12-10T07:15:03,026 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#B#compaction#95 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:03,027 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/7c423c3b85a544c3a8fdd071f41b5f22 is 50, key is test_row_0/B:col10/1733814901568/Put/seqid=0 2024-12-10T07:15:03,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741932_1108 (size=13391) 2024-12-10T07:15:03,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741933_1109 (size=13391) 2024-12-10T07:15:03,042 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/7c423c3b85a544c3a8fdd071f41b5f22 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/7c423c3b85a544c3a8fdd071f41b5f22 2024-12-10T07:15:03,049 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/B of 4204f23293292645e34a5d93616d8bf1 into 7c423c3b85a544c3a8fdd071f41b5f22(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:03,049 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:15:03,049 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/B, priority=12, startTime=1733814903009; duration=0sec 2024-12-10T07:15:03,049 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:03,049 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:B 2024-12-10T07:15:03,049 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:15:03,051 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:15:03,051 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 4204f23293292645e34a5d93616d8bf1/C is initiating minor compaction (all files) 2024-12-10T07:15:03,051 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4204f23293292645e34a5d93616d8bf1/C in TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:15:03,051 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/219329cfa2084c8986a5f7d0b828da95, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/1eacff445a2a4b46a4a6f4a896dc03ce, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/86e18e4056954f6bb0bd233b39caedb2, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/fa4cdefdbe7e49b184e8411caf659066] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp, totalSize=48.9 K 2024-12-10T07:15:03,051 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 219329cfa2084c8986a5f7d0b828da95, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733814898547 2024-12-10T07:15:03,052 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eacff445a2a4b46a4a6f4a896dc03ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733814898964 2024-12-10T07:15:03,052 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 86e18e4056954f6bb0bd233b39caedb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733814900284 2024-12-10T07:15:03,052 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting fa4cdefdbe7e49b184e8411caf659066, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733814901544 2024-12-10T07:15:03,068 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4204f23293292645e34a5d93616d8bf1#C#compaction#97 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:03,069 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/e304b89bb6884f0fa98b82f0400969a8 is 50, key is test_row_0/C:col10/1733814901568/Put/seqid=0 2024-12-10T07:15:03,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741934_1110 (size=13357) 2024-12-10T07:15:03,082 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/e304b89bb6884f0fa98b82f0400969a8 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/e304b89bb6884f0fa98b82f0400969a8 2024-12-10T07:15:03,088 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/C of 4204f23293292645e34a5d93616d8bf1 into e304b89bb6884f0fa98b82f0400969a8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:03,088 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:15:03,088 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/C, priority=12, startTime=1733814903009; duration=0sec 2024-12-10T07:15:03,088 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:03,088 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:C 2024-12-10T07:15:03,436 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/d62b2eb7a22c4f41b0dec5900bd3233b as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/d62b2eb7a22c4f41b0dec5900bd3233b 2024-12-10T07:15:03,442 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4204f23293292645e34a5d93616d8bf1/A of 4204f23293292645e34a5d93616d8bf1 into d62b2eb7a22c4f41b0dec5900bd3233b(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:03,442 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4204f23293292645e34a5d93616d8bf1: 2024-12-10T07:15:03,442 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., storeName=4204f23293292645e34a5d93616d8bf1/A, priority=12, startTime=1733814903009; duration=0sec 2024-12-10T07:15:03,443 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:03,443 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4204f23293292645e34a5d93616d8bf1:A 2024-12-10T07:15:03,708 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T07:15:10,450 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-10T07:15:10,450 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:15:10,450 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:10,450 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:10,450 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:15:10,451 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T07:15:10,451 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-12-10T07:15:10,451 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-12-10T07:15:10,451 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-12-10T07:15:10,451 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 152 2024-12-10T07:15:10,451 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-12-10T07:15:10,451 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T07:15:10,451 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2074 2024-12-10T07:15:10,451 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2047 2024-12-10T07:15:10,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T07:15:10,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1039 2024-12-10T07:15:10,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3109 rows 2024-12-10T07:15:10,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1038 2024-12-10T07:15:10,452 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3094 rows 2024-12-10T07:15:10,452 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T07:15:10,452 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T07:15:10,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:10,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:10,453 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:15:10,459 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-10T07:15:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T07:15:10,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-10T07:15:10,465 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733814910465"}]},"ts":"1733814910465"} 2024-12-10T07:15:10,467 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T07:15:10,467 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T07:15:10,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T07:15:10,474 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4204f23293292645e34a5d93616d8bf1, UNASSIGN}] 2024-12-10T07:15:10,475 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4204f23293292645e34a5d93616d8bf1, UNASSIGN 2024-12-10T07:15:10,477 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=4204f23293292645e34a5d93616d8bf1, regionState=CLOSING, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:10,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4204f23293292645e34a5d93616d8bf1, UNASSIGN because future has completed 2024-12-10T07:15:10,479 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T07:15:10,480 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835}] 2024-12-10T07:15:10,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-10T07:15:10,640 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:15:10,640 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T07:15:10,642 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 4204f23293292645e34a5d93616d8bf1, disabling compactions & flushes 2024-12-10T07:15:10,642 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:15:10,642 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:15:10,642 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. after waiting 0 ms 2024-12-10T07:15:10,642 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:15:10,643 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing 4204f23293292645e34a5d93616d8bf1 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-10T07:15:10,643 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=A 2024-12-10T07:15:10,643 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:10,643 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=B 2024-12-10T07:15:10,644 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:10,644 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4204f23293292645e34a5d93616d8bf1, store=C 2024-12-10T07:15:10,644 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:10,649 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/a163f977012e428b84fea80c3466a443 is 50, key is test_row_1/A:col10/1733814902205/Put/seqid=0 2024-12-10T07:15:10,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741935_1111 (size=9857) 2024-12-10T07:15:10,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-10T07:15:11,055 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/a163f977012e428b84fea80c3466a443 2024-12-10T07:15:11,063 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/2b94e3ec41b248f7b65693f61605a87e is 50, key is test_row_1/B:col10/1733814902205/Put/seqid=0 2024-12-10T07:15:11,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741936_1112 (size=9857) 2024-12-10T07:15:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-10T07:15:11,468 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/2b94e3ec41b248f7b65693f61605a87e 2024-12-10T07:15:11,483 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/f9533c01585b4ec694d473f1e9a18b70 is 50, key is test_row_1/C:col10/1733814902205/Put/seqid=0 2024-12-10T07:15:11,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741937_1113 (size=9857) 2024-12-10T07:15:11,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-10T07:15:11,888 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/f9533c01585b4ec694d473f1e9a18b70 2024-12-10T07:15:11,897 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/A/a163f977012e428b84fea80c3466a443 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/a163f977012e428b84fea80c3466a443 2024-12-10T07:15:11,902 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/a163f977012e428b84fea80c3466a443, entries=100, sequenceid=441, filesize=9.6 K 2024-12-10T07:15:11,902 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/B/2b94e3ec41b248f7b65693f61605a87e as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/2b94e3ec41b248f7b65693f61605a87e 2024-12-10T07:15:11,907 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/2b94e3ec41b248f7b65693f61605a87e, entries=100, sequenceid=441, filesize=9.6 K 2024-12-10T07:15:11,908 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/.tmp/C/f9533c01585b4ec694d473f1e9a18b70 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/f9533c01585b4ec694d473f1e9a18b70 2024-12-10T07:15:11,914 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/f9533c01585b4ec694d473f1e9a18b70, entries=100, sequenceid=441, filesize=9.6 K 2024-12-10T07:15:11,915 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 4204f23293292645e34a5d93616d8bf1 in 1273ms, sequenceid=441, compaction requested=false 2024-12-10T07:15:11,916 DEBUG [StoreCloser-TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/120f892f746441c788dd3754beb59ab6, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/fe4a3ec320bc4a838d4413a6028581bc, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/5db5ac87b5154299abaf186999e6c217, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/05d96eba6bad4d44b69c7cfd9fb7806a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/76e07534eb0c44d7b474c1f684f0ae70, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/8f0225770b004a8ebbf60893f4604d91, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/99249f607ae4473bb76969f33893314f, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/08e8d09a450c46f3bf722a817b064582, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/bc32f62739514cfd9ddeb393f2a0935a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/30d40f2ce79c43ff9a3714077a9cc445, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/388085c85a494a168a52ce60fedb3d54, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/52f80f7931594ac7b3cb7103f90ed548, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/21eeadb2f43446f2a5468d5d4b8695db, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/85cf3432970d43bbad6a098de4e520d9, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/e312cbbd5cb140f0ae1977310517cc1e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/1eb803766bf74c0da10f88eb02c2bafe, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/8a3c30f62d434009800efe8d86e583b6, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/a632a0b0f50c4c7e87b38b4a179cf513, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/0e59a1276841478bbe479869cd70c2b2, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/6b4eccffc24c4a619b5f2b846df97ff2, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/c2630c1735894aaf896b5587d5fa4592, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/3f6fbc5356354985b4040a42d3cbd731, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/71585b455e0c403b9664ce7173fc6156, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/0c7f0aec4aa5469193f0f4c2e6ebc54c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/8a3b14e702ae4136a6d06c0924566efc, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/f44bf120eb4a49208b98a096b8624425, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/5f79f3f5939e4db3820bf8cd7a8981d3, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/bd75bdbc39c243979ba803a812141de1, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/a8e6a28556044cc093b86288d6d84552, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/fcb0f2f688d040a9b6c7ecbb0ab7c82b, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/26759537b28749b4b433db1ca5d2028c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/52af774af51740deb684b099db1c0582] to archive 2024-12-10T07:15:11,919 DEBUG [StoreCloser-TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T07:15:11,925 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/8f0225770b004a8ebbf60893f4604d91 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/8f0225770b004a8ebbf60893f4604d91 2024-12-10T07:15:11,925 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/120f892f746441c788dd3754beb59ab6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/120f892f746441c788dd3754beb59ab6 2024-12-10T07:15:11,926 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/08e8d09a450c46f3bf722a817b064582 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/08e8d09a450c46f3bf722a817b064582 2024-12-10T07:15:11,926 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/99249f607ae4473bb76969f33893314f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/99249f607ae4473bb76969f33893314f 2024-12-10T07:15:11,926 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/fe4a3ec320bc4a838d4413a6028581bc to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/fe4a3ec320bc4a838d4413a6028581bc 2024-12-10T07:15:11,926 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/76e07534eb0c44d7b474c1f684f0ae70 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/76e07534eb0c44d7b474c1f684f0ae70 2024-12-10T07:15:11,926 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/05d96eba6bad4d44b69c7cfd9fb7806a to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/05d96eba6bad4d44b69c7cfd9fb7806a 2024-12-10T07:15:11,927 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/5db5ac87b5154299abaf186999e6c217 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/5db5ac87b5154299abaf186999e6c217 2024-12-10T07:15:11,929 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/bc32f62739514cfd9ddeb393f2a0935a to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/bc32f62739514cfd9ddeb393f2a0935a 2024-12-10T07:15:11,929 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/52f80f7931594ac7b3cb7103f90ed548 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/52f80f7931594ac7b3cb7103f90ed548 2024-12-10T07:15:11,929 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/21eeadb2f43446f2a5468d5d4b8695db to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/21eeadb2f43446f2a5468d5d4b8695db 2024-12-10T07:15:11,929 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/e312cbbd5cb140f0ae1977310517cc1e to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/e312cbbd5cb140f0ae1977310517cc1e 2024-12-10T07:15:11,930 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/30d40f2ce79c43ff9a3714077a9cc445 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/30d40f2ce79c43ff9a3714077a9cc445 2024-12-10T07:15:11,930 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/388085c85a494a168a52ce60fedb3d54 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/388085c85a494a168a52ce60fedb3d54 2024-12-10T07:15:11,930 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/85cf3432970d43bbad6a098de4e520d9 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/85cf3432970d43bbad6a098de4e520d9 2024-12-10T07:15:11,930 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/1eb803766bf74c0da10f88eb02c2bafe to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/1eb803766bf74c0da10f88eb02c2bafe 2024-12-10T07:15:11,932 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/0e59a1276841478bbe479869cd70c2b2 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/0e59a1276841478bbe479869cd70c2b2 2024-12-10T07:15:11,932 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/6b4eccffc24c4a619b5f2b846df97ff2 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/6b4eccffc24c4a619b5f2b846df97ff2 2024-12-10T07:15:11,932 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/3f6fbc5356354985b4040a42d3cbd731 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/3f6fbc5356354985b4040a42d3cbd731 2024-12-10T07:15:11,932 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/71585b455e0c403b9664ce7173fc6156 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/71585b455e0c403b9664ce7173fc6156 2024-12-10T07:15:11,932 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/0c7f0aec4aa5469193f0f4c2e6ebc54c to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/0c7f0aec4aa5469193f0f4c2e6ebc54c 2024-12-10T07:15:11,933 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/8a3c30f62d434009800efe8d86e583b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/8a3c30f62d434009800efe8d86e583b6 2024-12-10T07:15:11,933 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/c2630c1735894aaf896b5587d5fa4592 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/c2630c1735894aaf896b5587d5fa4592 2024-12-10T07:15:11,935 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/f44bf120eb4a49208b98a096b8624425 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/f44bf120eb4a49208b98a096b8624425 2024-12-10T07:15:11,935 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/8a3b14e702ae4136a6d06c0924566efc to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/8a3b14e702ae4136a6d06c0924566efc 2024-12-10T07:15:11,936 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/a632a0b0f50c4c7e87b38b4a179cf513 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/a632a0b0f50c4c7e87b38b4a179cf513 2024-12-10T07:15:11,936 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/5f79f3f5939e4db3820bf8cd7a8981d3 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/5f79f3f5939e4db3820bf8cd7a8981d3 2024-12-10T07:15:11,936 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/fcb0f2f688d040a9b6c7ecbb0ab7c82b to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/fcb0f2f688d040a9b6c7ecbb0ab7c82b 2024-12-10T07:15:11,936 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/bd75bdbc39c243979ba803a812141de1 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/bd75bdbc39c243979ba803a812141de1 2024-12-10T07:15:11,937 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/26759537b28749b4b433db1ca5d2028c to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/26759537b28749b4b433db1ca5d2028c 2024-12-10T07:15:11,937 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/a8e6a28556044cc093b86288d6d84552 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/a8e6a28556044cc093b86288d6d84552 2024-12-10T07:15:11,937 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/52af774af51740deb684b099db1c0582 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/52af774af51740deb684b099db1c0582 2024-12-10T07:15:11,948 DEBUG [StoreCloser-TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/8ed5f95145cd4dcab65c3728aa2d0340, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/14df0a8984c842d29827be370f1c0b5a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/df4db2c732d84fb49d47ea42d50080e6, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/2159d9cd57c1479ab441feb3eb29c73a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/d3727846339e415799a49c358d2bf782, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/c21641fa50be4963ac25d1b03972e66e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/d3cfbba2edb7406e8f8efec283952945, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/8e5c3aebdae748688488418e6b264de0, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/54782b43d2a64f62b9738d9af0b1f8ff, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/f0efd937b6a24c299784aaea83c8d868, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/437efa10cf3a461b9d33b620c2e7d435, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/4e95020c77f64cb9810487c1c2fb5e94, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/5d681fa896f0453d95df6ad0d7ea5e90, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/a2bf2f37c0ab4621a42691bb4fb0444b, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/d4bb173a0c28475a927b8f4c6b6dcc0e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/bcafcb93905b4881abe865cbd0c22230, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/8bd2cf5c7c984a40b2b575e50489b515, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/b473f35cc0b84141aa071308ea3d1177, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/fb85be95ca0a4ef0ba66ad6728df6fb0, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/e725a19575e3465eb2a98c506e558171, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/1fbdfe6d172749bf931492f55c3bfcbe, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/9eb1b05b456c4860832231b04ac0ae59, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/885c94819d644246823de91c668eff6d, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/59cb5f0e8ce84d85902eaeb9dded9775, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/19d61cae206148c586849302981b2b04, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/b4f669eae1bd49eeb54dc2841f6b7b77, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/d8d683600e6c43cdabb46733e10a282e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/7282959d44f74700b138d535ccc7e034, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/6de1ea3183be4e1eacdd907fa37f88bb, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/df907149234c4fe7990ab25683887373, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/14d907db65134a1cb3c55e2cbfbc133c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/3ddae56bcca24313ac09a02ddbdfb0b4] to archive 2024-12-10T07:15:11,950 DEBUG [StoreCloser-TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T07:15:11,952 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/8ed5f95145cd4dcab65c3728aa2d0340 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/8ed5f95145cd4dcab65c3728aa2d0340 2024-12-10T07:15:11,952 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/14df0a8984c842d29827be370f1c0b5a to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/14df0a8984c842d29827be370f1c0b5a 2024-12-10T07:15:11,952 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/d3727846339e415799a49c358d2bf782 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/d3727846339e415799a49c358d2bf782 2024-12-10T07:15:11,952 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/2159d9cd57c1479ab441feb3eb29c73a to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/2159d9cd57c1479ab441feb3eb29c73a 2024-12-10T07:15:11,952 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/8e5c3aebdae748688488418e6b264de0 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/8e5c3aebdae748688488418e6b264de0 2024-12-10T07:15:11,953 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/df4db2c732d84fb49d47ea42d50080e6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/df4db2c732d84fb49d47ea42d50080e6 2024-12-10T07:15:11,953 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/c21641fa50be4963ac25d1b03972e66e to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/c21641fa50be4963ac25d1b03972e66e 2024-12-10T07:15:11,953 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/d3cfbba2edb7406e8f8efec283952945 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/d3cfbba2edb7406e8f8efec283952945 2024-12-10T07:15:11,954 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/a2bf2f37c0ab4621a42691bb4fb0444b to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/a2bf2f37c0ab4621a42691bb4fb0444b 2024-12-10T07:15:11,955 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/4e95020c77f64cb9810487c1c2fb5e94 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/4e95020c77f64cb9810487c1c2fb5e94 2024-12-10T07:15:11,955 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/5d681fa896f0453d95df6ad0d7ea5e90 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/5d681fa896f0453d95df6ad0d7ea5e90 2024-12-10T07:15:11,955 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/54782b43d2a64f62b9738d9af0b1f8ff to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/54782b43d2a64f62b9738d9af0b1f8ff 2024-12-10T07:15:11,956 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/437efa10cf3a461b9d33b620c2e7d435 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/437efa10cf3a461b9d33b620c2e7d435 2024-12-10T07:15:11,956 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/f0efd937b6a24c299784aaea83c8d868 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/f0efd937b6a24c299784aaea83c8d868 2024-12-10T07:15:11,956 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/d4bb173a0c28475a927b8f4c6b6dcc0e to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/d4bb173a0c28475a927b8f4c6b6dcc0e 2024-12-10T07:15:11,956 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/bcafcb93905b4881abe865cbd0c22230 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/bcafcb93905b4881abe865cbd0c22230 2024-12-10T07:15:11,956 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/8bd2cf5c7c984a40b2b575e50489b515 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/8bd2cf5c7c984a40b2b575e50489b515 2024-12-10T07:15:11,957 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/fb85be95ca0a4ef0ba66ad6728df6fb0 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/fb85be95ca0a4ef0ba66ad6728df6fb0 2024-12-10T07:15:11,957 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/b473f35cc0b84141aa071308ea3d1177 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/b473f35cc0b84141aa071308ea3d1177 2024-12-10T07:15:11,958 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/9eb1b05b456c4860832231b04ac0ae59 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/9eb1b05b456c4860832231b04ac0ae59 2024-12-10T07:15:11,958 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/59cb5f0e8ce84d85902eaeb9dded9775 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/59cb5f0e8ce84d85902eaeb9dded9775 2024-12-10T07:15:11,958 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/e725a19575e3465eb2a98c506e558171 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/e725a19575e3465eb2a98c506e558171 2024-12-10T07:15:11,958 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/885c94819d644246823de91c668eff6d to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/885c94819d644246823de91c668eff6d 2024-12-10T07:15:11,958 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/1fbdfe6d172749bf931492f55c3bfcbe to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/1fbdfe6d172749bf931492f55c3bfcbe 2024-12-10T07:15:11,959 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/19d61cae206148c586849302981b2b04 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/19d61cae206148c586849302981b2b04 2024-12-10T07:15:11,959 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/b4f669eae1bd49eeb54dc2841f6b7b77 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/b4f669eae1bd49eeb54dc2841f6b7b77 2024-12-10T07:15:11,959 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/d8d683600e6c43cdabb46733e10a282e to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/d8d683600e6c43cdabb46733e10a282e 2024-12-10T07:15:11,960 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/df907149234c4fe7990ab25683887373 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/df907149234c4fe7990ab25683887373 2024-12-10T07:15:11,960 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/7282959d44f74700b138d535ccc7e034 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/7282959d44f74700b138d535ccc7e034 2024-12-10T07:15:11,960 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/6de1ea3183be4e1eacdd907fa37f88bb to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/6de1ea3183be4e1eacdd907fa37f88bb 2024-12-10T07:15:11,960 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/3ddae56bcca24313ac09a02ddbdfb0b4 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/3ddae56bcca24313ac09a02ddbdfb0b4 2024-12-10T07:15:11,961 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/14d907db65134a1cb3c55e2cbfbc133c to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/14d907db65134a1cb3c55e2cbfbc133c 2024-12-10T07:15:11,962 DEBUG [StoreCloser-TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/10fa3f411fdb4dfbabce05ebc4da3add, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/a5bee603dde64bfdb34536aacdc1f700, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/f6ecf8c294204d38ba7ce6c2668e8683, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/728b0308e00c41dbac804aeb70cad886, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/1ee66c102ef541d4a741c553dd138194, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/4c8c387b95c9486fabff3915d6d39777, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/a8602f8876a84bc3b1921b552477c446, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/4dd51fc60b8b41928e8475639c9eec8c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/4ff7252a261b4db694c16b8501d4f79a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/367d918a5d834d5e9acb4fead9c291a1, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/ae6bb01ff79a46ee8b2a896dde96677e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/cf46de35e7214f28a884f3e8ce19afa0, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/6877685e2960444186c6bb2e9e48f3ab, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/f1c8732544d74f4f8f0f1dacb44bbdc4, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/07929e3f1a1b45c5b76aa271be9c0920, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/e20d09f87a344bb5a65b63d92d9c4bca, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/299b8c6b31a74d21a61e95845a5453b5, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/bc4ea33f50094cbf9a21295c4a6ac6b4, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/fe8408da8c6f4dca8ca5d2d76fbae385, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/008b6be261e148f896616862beb71aa0, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/183c870f5b0141d1a3fc1668f3f9f324, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/79047079ebe64c2da1371a87f5ff3aa2, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/18649dc3bae44c52aaaf2e2b784972fe, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/4c736fcdab31489f9c20aa0da5cc2e1d, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/0175dc1ed9674896982ca384359bf784, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/863631ef5d9d4c218c9c2cfb0c893129, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/219329cfa2084c8986a5f7d0b828da95, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/238e32d427f6466e873ff8b1bb1da0e9, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/1eacff445a2a4b46a4a6f4a896dc03ce, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/86e18e4056954f6bb0bd233b39caedb2, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/fa4cdefdbe7e49b184e8411caf659066] to archive 2024-12-10T07:15:11,963 DEBUG [StoreCloser-TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T07:15:11,965 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/728b0308e00c41dbac804aeb70cad886 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/728b0308e00c41dbac804aeb70cad886 2024-12-10T07:15:11,965 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/a5bee603dde64bfdb34536aacdc1f700 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/a5bee603dde64bfdb34536aacdc1f700 2024-12-10T07:15:11,965 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/10fa3f411fdb4dfbabce05ebc4da3add to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/10fa3f411fdb4dfbabce05ebc4da3add 2024-12-10T07:15:11,965 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/f6ecf8c294204d38ba7ce6c2668e8683 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/f6ecf8c294204d38ba7ce6c2668e8683 2024-12-10T07:15:11,967 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/1ee66c102ef541d4a741c553dd138194 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/1ee66c102ef541d4a741c553dd138194 2024-12-10T07:15:11,967 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/4c8c387b95c9486fabff3915d6d39777 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/4c8c387b95c9486fabff3915d6d39777 2024-12-10T07:15:11,968 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/367d918a5d834d5e9acb4fead9c291a1 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/367d918a5d834d5e9acb4fead9c291a1 2024-12-10T07:15:11,968 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/a8602f8876a84bc3b1921b552477c446 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/a8602f8876a84bc3b1921b552477c446 2024-12-10T07:15:11,968 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/ae6bb01ff79a46ee8b2a896dde96677e to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/ae6bb01ff79a46ee8b2a896dde96677e 2024-12-10T07:15:11,968 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/4ff7252a261b4db694c16b8501d4f79a to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/4ff7252a261b4db694c16b8501d4f79a 2024-12-10T07:15:11,969 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/4dd51fc60b8b41928e8475639c9eec8c to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/4dd51fc60b8b41928e8475639c9eec8c 2024-12-10T07:15:11,969 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/cf46de35e7214f28a884f3e8ce19afa0 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/cf46de35e7214f28a884f3e8ce19afa0 2024-12-10T07:15:11,969 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/6877685e2960444186c6bb2e9e48f3ab to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/6877685e2960444186c6bb2e9e48f3ab 2024-12-10T07:15:11,970 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/f1c8732544d74f4f8f0f1dacb44bbdc4 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/f1c8732544d74f4f8f0f1dacb44bbdc4 2024-12-10T07:15:11,971 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/07929e3f1a1b45c5b76aa271be9c0920 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/07929e3f1a1b45c5b76aa271be9c0920 2024-12-10T07:15:11,971 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/e20d09f87a344bb5a65b63d92d9c4bca to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/e20d09f87a344bb5a65b63d92d9c4bca 2024-12-10T07:15:11,971 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/bc4ea33f50094cbf9a21295c4a6ac6b4 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/bc4ea33f50094cbf9a21295c4a6ac6b4 2024-12-10T07:15:11,971 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/183c870f5b0141d1a3fc1668f3f9f324 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/183c870f5b0141d1a3fc1668f3f9f324 2024-12-10T07:15:11,972 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/299b8c6b31a74d21a61e95845a5453b5 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/299b8c6b31a74d21a61e95845a5453b5 2024-12-10T07:15:11,972 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/008b6be261e148f896616862beb71aa0 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/008b6be261e148f896616862beb71aa0 2024-12-10T07:15:11,973 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/fe8408da8c6f4dca8ca5d2d76fbae385 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/fe8408da8c6f4dca8ca5d2d76fbae385 2024-12-10T07:15:11,973 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/18649dc3bae44c52aaaf2e2b784972fe to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/18649dc3bae44c52aaaf2e2b784972fe 2024-12-10T07:15:11,973 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/79047079ebe64c2da1371a87f5ff3aa2 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/79047079ebe64c2da1371a87f5ff3aa2 2024-12-10T07:15:11,973 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/863631ef5d9d4c218c9c2cfb0c893129 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/863631ef5d9d4c218c9c2cfb0c893129 2024-12-10T07:15:11,974 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/0175dc1ed9674896982ca384359bf784 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/0175dc1ed9674896982ca384359bf784 2024-12-10T07:15:11,974 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/4c736fcdab31489f9c20aa0da5cc2e1d to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/4c736fcdab31489f9c20aa0da5cc2e1d 2024-12-10T07:15:11,974 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/219329cfa2084c8986a5f7d0b828da95 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/219329cfa2084c8986a5f7d0b828da95 2024-12-10T07:15:11,975 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/238e32d427f6466e873ff8b1bb1da0e9 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/238e32d427f6466e873ff8b1bb1da0e9 2024-12-10T07:15:11,975 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/1eacff445a2a4b46a4a6f4a896dc03ce to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/1eacff445a2a4b46a4a6f4a896dc03ce 2024-12-10T07:15:11,975 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/86e18e4056954f6bb0bd233b39caedb2 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/86e18e4056954f6bb0bd233b39caedb2 2024-12-10T07:15:11,975 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/fa4cdefdbe7e49b184e8411caf659066 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/fa4cdefdbe7e49b184e8411caf659066 2024-12-10T07:15:11,981 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/recovered.edits/444.seqid, newMaxSeqId=444, maxSeqId=1 2024-12-10T07:15:11,982 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. 2024-12-10T07:15:11,982 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 4204f23293292645e34a5d93616d8bf1: Waiting for close lock at 1733814910642Running coprocessor pre-close hooks at 1733814910642Disabling compacts and flushes for region at 1733814910642Disabling writes for close at 1733814910642Obtaining lock to block concurrent updates at 1733814910643 (+1 ms)Preparing flush snapshotting stores in 4204f23293292645e34a5d93616d8bf1 at 1733814910643Finished memstore snapshotting TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733814910644 (+1 ms)Flushing stores of TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1. at 1733814910645 (+1 ms)Flushing 4204f23293292645e34a5d93616d8bf1/A: creating writer at 1733814910645Flushing 4204f23293292645e34a5d93616d8bf1/A: appending metadata at 1733814910649 (+4 ms)Flushing 4204f23293292645e34a5d93616d8bf1/A: closing flushed file at 1733814910649Flushing 4204f23293292645e34a5d93616d8bf1/B: creating writer at 1733814911060 (+411 ms)Flushing 4204f23293292645e34a5d93616d8bf1/B: appending metadata at 1733814911062 (+2 ms)Flushing 4204f23293292645e34a5d93616d8bf1/B: closing flushed file at 1733814911062Flushing 4204f23293292645e34a5d93616d8bf1/C: creating writer at 1733814911479 (+417 ms)Flushing 4204f23293292645e34a5d93616d8bf1/C: appending metadata at 1733814911482 (+3 ms)Flushing 4204f23293292645e34a5d93616d8bf1/C: closing flushed file at 1733814911482Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@105e6dba: reopening flushed file at 1733814911896 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ba7ecb3: reopening flushed file at 1733814911902 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2203db23: reopening flushed file at 1733814911907 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 4204f23293292645e34a5d93616d8bf1 in 1273ms, sequenceid=441, compaction requested=false at 1733814911915 (+8 ms)Writing region close event to WAL at 1733814911976 (+61 ms)Running coprocessor post-close hooks at 1733814911982 (+6 ms)Closed at 1733814911982 2024-12-10T07:15:11,985 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 4204f23293292645e34a5d93616d8bf1 2024-12-10T07:15:11,985 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=4204f23293292645e34a5d93616d8bf1, regionState=CLOSED 2024-12-10T07:15:11,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 because future has completed 2024-12-10T07:15:11,990 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-10T07:15:11,990 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 4204f23293292645e34a5d93616d8bf1, server=b1396b8d2d12,35871,1733814875835 in 1.5080 sec 2024-12-10T07:15:11,992 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-10T07:15:11,992 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4204f23293292645e34a5d93616d8bf1, UNASSIGN in 1.5160 sec 2024-12-10T07:15:11,994 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-10T07:15:11,995 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5240 sec 2024-12-10T07:15:11,996 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733814911996"}]},"ts":"1733814911996"} 2024-12-10T07:15:11,998 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T07:15:11,998 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T07:15:11,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5380 sec 2024-12-10T07:15:12,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-10T07:15:12,615 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:15:12,618 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-10T07:15:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:15:12,623 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:15:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-10T07:15:12,625 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:15:12,630 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1 2024-12-10T07:15:12,634 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A, FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B, FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C, FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/recovered.edits] 2024-12-10T07:15:12,638 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/a163f977012e428b84fea80c3466a443 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/a163f977012e428b84fea80c3466a443 2024-12-10T07:15:12,639 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/d62b2eb7a22c4f41b0dec5900bd3233b to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/A/d62b2eb7a22c4f41b0dec5900bd3233b 2024-12-10T07:15:12,642 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/7c423c3b85a544c3a8fdd071f41b5f22 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/7c423c3b85a544c3a8fdd071f41b5f22 2024-12-10T07:15:12,643 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/2b94e3ec41b248f7b65693f61605a87e to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/B/2b94e3ec41b248f7b65693f61605a87e 2024-12-10T07:15:12,646 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/f9533c01585b4ec694d473f1e9a18b70 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/f9533c01585b4ec694d473f1e9a18b70 2024-12-10T07:15:12,646 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/e304b89bb6884f0fa98b82f0400969a8 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/C/e304b89bb6884f0fa98b82f0400969a8 2024-12-10T07:15:12,649 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/recovered.edits/444.seqid to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1/recovered.edits/444.seqid 2024-12-10T07:15:12,650 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/4204f23293292645e34a5d93616d8bf1 2024-12-10T07:15:12,650 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T07:15:12,655 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:15:12,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-10T07:15:12,665 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T07:15:12,698 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T07:15:12,700 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:15:12,700 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T07:15:12,701 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733814912700"}]},"ts":"9223372036854775807"} 2024-12-10T07:15:12,705 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T07:15:12,705 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4204f23293292645e34a5d93616d8bf1, NAME => 'TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T07:15:12,705 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T07:15:12,706 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733814912705"}]},"ts":"9223372036854775807"} 2024-12-10T07:15:12,709 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T07:15:12,710 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:15:12,713 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 91 msec 2024-12-10T07:15:12,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-10T07:15:12,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T07:15:12,735 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:15:12,749 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=250 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-673173346_22 at /127.0.0.1:40984 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;b1396b8d2d12:35871-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=829 (was 716) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3403 (was 4078) 2024-12-10T07:15:12,760 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=829, ProcessCount=11, AvailableMemoryMB=3403 2024-12-10T07:15:12,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T07:15:12,763 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T07:15:12,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T07:15:12,770 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T07:15:12,770 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:12,770 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-12-10T07:15:12,771 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T07:15:12,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-10T07:15:12,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741938_1114 (size=1061) 2024-12-10T07:15:12,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-10T07:15:13,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-10T07:15:13,193 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8 2024-12-10T07:15:13,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741939_1115 (size=53) 2024-12-10T07:15:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-10T07:15:13,601 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T07:15:13,601 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 573523ff7c74144a068617629803d4b6, disabling compactions & flushes 2024-12-10T07:15:13,601 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:13,602 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:13,602 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. after waiting 0 ms 2024-12-10T07:15:13,602 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:13,602 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:13,602 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 573523ff7c74144a068617629803d4b6: Waiting for close lock at 1733814913601Disabling compacts and flushes for region at 1733814913601Disabling writes for close at 1733814913602 (+1 ms)Writing region close event to WAL at 1733814913602Closed at 1733814913602 2024-12-10T07:15:13,603 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T07:15:13,603 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733814913603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733814913603"}]},"ts":"1733814913603"} 2024-12-10T07:15:13,605 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T07:15:13,606 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T07:15:13,606 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733814913606"}]},"ts":"1733814913606"} 2024-12-10T07:15:13,608 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T07:15:13,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=573523ff7c74144a068617629803d4b6, ASSIGN}] 2024-12-10T07:15:13,610 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=573523ff7c74144a068617629803d4b6, ASSIGN 2024-12-10T07:15:13,610 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=573523ff7c74144a068617629803d4b6, ASSIGN; state=OFFLINE, location=b1396b8d2d12,35871,1733814875835; forceNewPlan=false, retain=false 2024-12-10T07:15:13,761 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=573523ff7c74144a068617629803d4b6, regionState=OPENING, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:13,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=573523ff7c74144a068617629803d4b6, ASSIGN because future has completed 2024-12-10T07:15:13,764 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835}] 2024-12-10T07:15:13,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-10T07:15:13,919 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:13,919 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} 2024-12-10T07:15:13,920 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:13,920 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T07:15:13,920 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:13,920 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:13,922 INFO [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:13,923 INFO [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:15:13,923 INFO [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 573523ff7c74144a068617629803d4b6 columnFamilyName A 2024-12-10T07:15:13,924 DEBUG [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:13,924 INFO [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] regionserver.HStore(327): Store=573523ff7c74144a068617629803d4b6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:15:13,924 INFO [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:13,925 INFO [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:15:13,926 INFO [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 573523ff7c74144a068617629803d4b6 columnFamilyName B 2024-12-10T07:15:13,926 DEBUG [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:13,926 INFO [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] regionserver.HStore(327): Store=573523ff7c74144a068617629803d4b6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:15:13,926 INFO [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:13,928 INFO [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:15:13,928 INFO [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 573523ff7c74144a068617629803d4b6 columnFamilyName C 2024-12-10T07:15:13,929 DEBUG [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:13,929 INFO [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] regionserver.HStore(327): Store=573523ff7c74144a068617629803d4b6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:15:13,929 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:13,929 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:13,930 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6 2024-12-10T07:15:13,930 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6 2024-12-10T07:15:13,931 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:13,931 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:13,932 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T07:15:13,933 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:13,936 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T07:15:13,937 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 573523ff7c74144a068617629803d4b6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59055846, jitterRate=-0.11999931931495667}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T07:15:13,937 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:13,938 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 573523ff7c74144a068617629803d4b6: Running coprocessor pre-open hook at 1733814913920Writing region info on filesystem at 1733814913920Initializing all the Stores at 1733814913921 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733814913921Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733814913921Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733814913921Cleaning up temporary data from old regions at 1733814913931 (+10 ms)Running coprocessor post-open hooks at 1733814913937 (+6 ms)Region opened successfully at 1733814913938 (+1 ms) 2024-12-10T07:15:13,939 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., pid=32, masterSystemTime=1733814913916 2024-12-10T07:15:13,942 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:13,942 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:13,943 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=573523ff7c74144a068617629803d4b6, regionState=OPEN, openSeqNum=2, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:13,945 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 because future has completed 2024-12-10T07:15:13,948 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-10T07:15:13,948 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 in 182 msec 2024-12-10T07:15:13,951 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-10T07:15:13,951 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=573523ff7c74144a068617629803d4b6, ASSIGN in 340 msec 2024-12-10T07:15:13,952 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T07:15:13,952 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733814913952"}]},"ts":"1733814913952"} 2024-12-10T07:15:13,954 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T07:15:13,955 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T07:15:13,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1920 sec 2024-12-10T07:15:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-10T07:15:14,915 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:15:14,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@110f9666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:14,919 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:15:14,919 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:15:14,920 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:15:14,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:15:14,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:15:14,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7be46ed5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:14,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:15:14,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:15:14,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:14,922 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:15:14,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79906bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:14,924 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:15:14,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:15:14,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:15:14,927 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:15:14,928 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T07:15:14,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is b1396b8d2d12,36259,1733814874775 2024-12-10T07:15:14,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@16e27d07 2024-12-10T07:15:14,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T07:15:14,930 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T07:15:14,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T07:15:14,935 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T07:15:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-10T07:15:14,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741940_1116 (size=1097) 2024-12-10T07:15:15,372 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-10T07:15:15,372 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-10T07:15:15,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T07:15:15,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=573523ff7c74144a068617629803d4b6, REOPEN/MOVE}] 2024-12-10T07:15:15,383 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=573523ff7c74144a068617629803d4b6, REOPEN/MOVE 2024-12-10T07:15:15,383 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=573523ff7c74144a068617629803d4b6, regionState=CLOSING, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:15,385 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=573523ff7c74144a068617629803d4b6, REOPEN/MOVE because future has completed 2024-12-10T07:15:15,386 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T07:15:15,386 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835}] 2024-12-10T07:15:15,538 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:15,538 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T07:15:15,538 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 573523ff7c74144a068617629803d4b6, disabling compactions & flushes 2024-12-10T07:15:15,538 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:15,538 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:15,538 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. after waiting 0 ms 2024-12-10T07:15:15,539 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:15,542 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-10T07:15:15,543 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:15,543 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 573523ff7c74144a068617629803d4b6: Waiting for close lock at 1733814915538Running coprocessor pre-close hooks at 1733814915538Disabling compacts and flushes for region at 1733814915538Disabling writes for close at 1733814915539 (+1 ms)Writing region close event to WAL at 1733814915539Running coprocessor post-close hooks at 1733814915543 (+4 ms)Closed at 1733814915543 2024-12-10T07:15:15,543 WARN [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 573523ff7c74144a068617629803d4b6 to self. 2024-12-10T07:15:15,545 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:15,546 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=573523ff7c74144a068617629803d4b6, regionState=CLOSED 2024-12-10T07:15:15,547 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 because future has completed 2024-12-10T07:15:15,550 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-10T07:15:15,550 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 in 162 msec 2024-12-10T07:15:15,551 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=573523ff7c74144a068617629803d4b6, REOPEN/MOVE; state=CLOSED, location=b1396b8d2d12,35871,1733814875835; forceNewPlan=false, retain=true 2024-12-10T07:15:15,701 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=573523ff7c74144a068617629803d4b6, regionState=OPENING, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:15,703 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=573523ff7c74144a068617629803d4b6, REOPEN/MOVE because future has completed 2024-12-10T07:15:15,703 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835}] 2024-12-10T07:15:15,859 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:15,859 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} 2024-12-10T07:15:15,860 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:15,860 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T07:15:15,860 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:15,860 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:15,865 INFO [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:15,866 INFO [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:15:15,871 INFO [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 573523ff7c74144a068617629803d4b6 columnFamilyName A 2024-12-10T07:15:15,874 DEBUG [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:15,874 INFO [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] regionserver.HStore(327): Store=573523ff7c74144a068617629803d4b6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:15:15,875 INFO [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:15,876 INFO [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:15:15,876 INFO [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 573523ff7c74144a068617629803d4b6 columnFamilyName B 2024-12-10T07:15:15,876 DEBUG [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:15,877 INFO [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] regionserver.HStore(327): Store=573523ff7c74144a068617629803d4b6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:15:15,877 INFO [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:15,877 INFO [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:15:15,878 INFO [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 573523ff7c74144a068617629803d4b6 columnFamilyName C 2024-12-10T07:15:15,878 DEBUG [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:15,878 INFO [StoreOpener-573523ff7c74144a068617629803d4b6-1 {}] regionserver.HStore(327): Store=573523ff7c74144a068617629803d4b6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:15:15,879 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:15,879 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:15,879 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6 2024-12-10T07:15:15,880 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6 2024-12-10T07:15:15,881 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:15,881 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:15,882 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T07:15:15,883 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:15,884 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 573523ff7c74144a068617629803d4b6; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73786878, jitterRate=0.09951016306877136}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T07:15:15,884 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:15,887 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 573523ff7c74144a068617629803d4b6: Running coprocessor pre-open hook at 1733814915860Writing region info on filesystem at 1733814915860Initializing all the Stores at 1733814915861 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733814915861Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733814915861Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733814915861Cleaning up temporary data from old regions at 1733814915881 (+20 ms)Running coprocessor post-open hooks at 1733814915884 (+3 ms)Region opened successfully at 1733814915887 (+3 ms) 2024-12-10T07:15:15,888 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., pid=37, masterSystemTime=1733814915855 2024-12-10T07:15:15,890 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:15,891 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:15,891 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=573523ff7c74144a068617629803d4b6, regionState=OPEN, openSeqNum=5, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:15,893 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 because future has completed 2024-12-10T07:15:15,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-12-10T07:15:15,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 in 191 msec 2024-12-10T07:15:15,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-10T07:15:15,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=573523ff7c74144a068617629803d4b6, REOPEN/MOVE in 516 msec 2024-12-10T07:15:15,906 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-10T07:15:15,906 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 528 msec 2024-12-10T07:15:15,909 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 970 msec 2024-12-10T07:15:15,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-10T07:15:15,911 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:15:15,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e0e4267, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,912 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:15:15,912 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:15:15,913 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:15:15,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:15:15,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:15:15,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9cd281, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:15:15,914 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:15:15,914 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:15,915 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:15:15,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3310dfa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dc23ca6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,917 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:15:15,917 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:15:15,918 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:15:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:15:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:15:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a3b5484, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:15:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:15:15,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:15,920 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:15:15,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24fc69b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,922 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@636d090d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,922 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:15:15,922 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:15:15,923 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:15:15,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:15:15,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:15:15,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4500b092, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:15:15,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:15:15,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:15,925 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:15:15,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@241cd17f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63d8f52d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,928 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:15:15,928 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:15:15,930 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:15:15,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:15:15,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:15:15,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2367661f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:15:15,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:15:15,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:15,932 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:15:15,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23be8464, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0d456e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,934 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:15:15,934 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:15:15,935 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:15:15,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:15:15,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:15:15,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56f034e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:15:15,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:15:15,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:15,937 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:15:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@376c4eb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,942 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27c26b5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,942 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:15:15,942 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:15:15,943 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:15:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:15:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:15:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c692da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:15:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:15:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:15,945 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38644, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:15:15,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ea1a2c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30f10c2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,949 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:15:15,949 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:15:15,950 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:15:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:15:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:15:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e6f489, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:15:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:15:15,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:15,951 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:15:15,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f6c58b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,953 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7adec721, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,953 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:15:15,954 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:15:15,954 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:15:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:15:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:15:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71d6fde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:15:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:15:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:15,956 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:15:15,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@721efe8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,961 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78c2fc25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,961 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:15:15,961 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:15:15,962 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:15:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:15:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:15:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56755949, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:15:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:15:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:15,964 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:15:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c31ef4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:15,966 DEBUG [Thread-605 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:15:15,966 DEBUG [Thread-603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:15:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:15:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:15:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:15:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:15:15,968 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:15:15,971 DEBUG [Thread-601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:15:15,971 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:15:15,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:15:15,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:15:15,973 DEBUG [Thread-608 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:15:15,973 DEBUG [Thread-612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:15:15,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:15:15,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:15:15,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:15:15,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:15:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:15:15,977 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:15:15,978 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:15:15,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:15:15,979 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:15:15,981 DEBUG [Thread-597 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:15:15,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:15:15,982 DEBUG [Thread-610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:15:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:15:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:15:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:15:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:15:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:15:15,986 DEBUG [Thread-599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:15:15,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:15:15,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:15:15,989 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:15:15,989 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:15:15,989 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:15:15,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:15:15,991 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:15:15,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-12-10T07:15:15,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-10T07:15:15,994 DEBUG [Thread-614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:15:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:15:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:15:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:15:15,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:15:15,997 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:15:15,999 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:15:15,999 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:15:16,000 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:15:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:15:16,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:15:16,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:16,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 573523ff7c74144a068617629803d4b6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T07:15:16,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=A 2024-12-10T07:15:16,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:16,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=B 2024-12-10T07:15:16,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:16,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=C 2024-12-10T07:15:16,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:16,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:16,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814976081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:16,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:16,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:16,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-10T07:15:16,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814976089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:16,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733814916068, max timestamp=1733814916089Num uniques 859; 2024-12-10T07:15:16,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T07:15:16,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:16,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733814916068, max timestamp=1733814916098Num uniques 859; 2024-12-10T07:15:16,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T07:15:16,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733814916068, max timestamp=1733814916089Num uniques 859; 2024-12-10T07:15:16,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T07:15:16,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T07:15:16,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:16,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:16,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:16,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:16,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:16,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121064e3accd2da94e209605f0c76d77c8ce_573523ff7c74144a068617629803d4b6 is 50, key is test_row_0/A:col10/1733814915997/Put/seqid=0 2024-12-10T07:15:16,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814976216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:16,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741941_1117 (size=12154) 2024-12-10T07:15:16,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:16,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:16,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814976216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:16,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:16,237 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:16,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814976225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:16,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:16,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814976232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:16,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:16,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:16,267 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121064e3accd2da94e209605f0c76d77c8ce_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121064e3accd2da94e209605f0c76d77c8ce_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814976243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:16,270 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/31e0933d216248ef8cbccd89a28a65e9, store: [table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:16,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/31e0933d216248ef8cbccd89a28a65e9 is 175, key is test_row_0/A:col10/1733814915997/Put/seqid=0 2024-12-10T07:15:16,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T07:15:16,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:16,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:16,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:16,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:16,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:16,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:16,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-10T07:15:16,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741942_1118 (size=30955) 2024-12-10T07:15:16,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:16,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814976345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:16,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:16,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814976345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:16,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:16,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:16,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814976365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:16,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:16,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814976375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:16,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:16,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814976465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:16,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:16,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T07:15:16,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:16,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:16,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:16,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:16,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:16,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:16,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814976561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:16,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:16,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814976573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:16,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:16,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:16,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814976577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:16,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:16,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:16,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814976585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:16,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:16,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T07:15:16,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:16,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:16,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:16,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:16,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:16,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:16,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-10T07:15:16,718 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/31e0933d216248ef8cbccd89a28a65e9 2024-12-10T07:15:16,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/4f639267d7a24e9380441020435d4ca0 is 50, key is test_row_0/B:col10/1733814915997/Put/seqid=0 2024-12-10T07:15:16,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T07:15:16,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:16,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:16,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:16,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:16,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:16,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:16,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:16,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814976779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:16,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741943_1119 (size=12001) 2024-12-10T07:15:16,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/4f639267d7a24e9380441020435d4ca0 2024-12-10T07:15:16,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814976874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:16,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:16,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:16,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814976886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:16,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:16,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814976886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:16,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:16,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:16,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/71360b8471704734a8a36a1bc822f79c is 50, key is test_row_0/C:col10/1733814915997/Put/seqid=0 2024-12-10T07:15:16,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:16,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814976906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:16,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:16,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:16,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T07:15:16,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:16,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:16,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:16,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:16,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:16,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:16,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741944_1120 (size=12001) 2024-12-10T07:15:17,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T07:15:17,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:17,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:17,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:17,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:17,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-10T07:15:17,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T07:15:17,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:17,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:17,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:17,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:17,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:17,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:17,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:17,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814977301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:17,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:17,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:17,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:17,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/71360b8471704734a8a36a1bc822f79c 2024-12-10T07:15:17,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/31e0933d216248ef8cbccd89a28a65e9 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/31e0933d216248ef8cbccd89a28a65e9 2024-12-10T07:15:17,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/31e0933d216248ef8cbccd89a28a65e9, entries=150, sequenceid=14, filesize=30.2 K 2024-12-10T07:15:17,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/4f639267d7a24e9380441020435d4ca0 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/4f639267d7a24e9380441020435d4ca0 2024-12-10T07:15:17,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:17,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814977389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:17,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:17,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:17,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:17,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/4f639267d7a24e9380441020435d4ca0, entries=150, sequenceid=14, filesize=11.7 K 2024-12-10T07:15:17,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/71360b8471704734a8a36a1bc822f79c as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/71360b8471704734a8a36a1bc822f79c 2024-12-10T07:15:17,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:17,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T07:15:17,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:17,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:17,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:17,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:17,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814977397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:17,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:17,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:17,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814977397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:17,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:17,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:17,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:17,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:17,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/71360b8471704734a8a36a1bc822f79c, entries=150, sequenceid=14, filesize=11.7 K 2024-12-10T07:15:17,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 573523ff7c74144a068617629803d4b6 in 1411ms, sequenceid=14, compaction requested=false 2024-12-10T07:15:17,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:17,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:17,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 573523ff7c74144a068617629803d4b6 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-10T07:15:17,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=A 2024-12-10T07:15:17,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T07:15:17,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=B 2024-12-10T07:15:17,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T07:15:17,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=C 2024-12-10T07:15:17,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T07:15:17,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121084e3406923704a7f80908b1e6734b8b8_573523ff7c74144a068617629803d4b6 is 50, key is test_row_0/A:col10/1733814917433/Put/seqid=0 2024-12-10T07:15:17,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741945_1121 (size=14594) 2024-12-10T07:15:17,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T07:15:17,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:17,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:17,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:17,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:17,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:17,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:17,593 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T07:15:17,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:17,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814977599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:17,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:17,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:17,613 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:17,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T07:15:17,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:17,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814977717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:17,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:17,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:17,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:17,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:17,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:17,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T07:15:17,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:17,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:17,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:17,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:17,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:17,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814977945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:17,968 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:17,977 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121084e3406923704a7f80908b1e6734b8b8_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121084e3406923704a7f80908b1e6734b8b8_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:17,980 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/50dc2eae10534b36bbb1881390225a95, store: [table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:17,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/50dc2eae10534b36bbb1881390225a95 is 175, key is test_row_0/A:col10/1733814917433/Put/seqid=0 2024-12-10T07:15:18,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741946_1122 (size=39549) 2024-12-10T07:15:18,014 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/50dc2eae10534b36bbb1881390225a95 2024-12-10T07:15:18,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/16b23a711b8545c6aa4891c53aa5d204 is 50, key is test_row_0/B:col10/1733814917433/Put/seqid=0 2024-12-10T07:15:18,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T07:15:18,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:18,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:18,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:18,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:18,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:18,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741947_1123 (size=12001) 2024-12-10T07:15:18,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/16b23a711b8545c6aa4891c53aa5d204 2024-12-10T07:15:18,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/af68ca007b374c119f90cb4685a9655e is 50, key is test_row_0/C:col10/1733814917433/Put/seqid=0 2024-12-10T07:15:18,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741948_1124 (size=12001) 2024-12-10T07:15:18,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/af68ca007b374c119f90cb4685a9655e 2024-12-10T07:15:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-10T07:15:18,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/50dc2eae10534b36bbb1881390225a95 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/50dc2eae10534b36bbb1881390225a95 2024-12-10T07:15:18,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/50dc2eae10534b36bbb1881390225a95, entries=200, sequenceid=52, filesize=38.6 K 2024-12-10T07:15:18,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/16b23a711b8545c6aa4891c53aa5d204 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/16b23a711b8545c6aa4891c53aa5d204 2024-12-10T07:15:18,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T07:15:18,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:18,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:18,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:18,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:18,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:18,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/16b23a711b8545c6aa4891c53aa5d204, entries=150, sequenceid=52, filesize=11.7 K 2024-12-10T07:15:18,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/af68ca007b374c119f90cb4685a9655e as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/af68ca007b374c119f90cb4685a9655e 2024-12-10T07:15:18,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/af68ca007b374c119f90cb4685a9655e, entries=150, sequenceid=52, filesize=11.7 K 2024-12-10T07:15:18,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 573523ff7c74144a068617629803d4b6 in 786ms, sequenceid=52, compaction requested=false 2024-12-10T07:15:18,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:18,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 573523ff7c74144a068617629803d4b6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T07:15:18,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=A 2024-12-10T07:15:18,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:18,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=B 2024-12-10T07:15:18,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:18,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=C 2024-12-10T07:15:18,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:18,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210969ef5dbba4d4a3086077ff524de909b_573523ff7c74144a068617629803d4b6 is 50, key is test_row_0/A:col10/1733814917547/Put/seqid=0 2024-12-10T07:15:18,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741949_1125 (size=14594) 2024-12-10T07:15:18,347 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:18,355 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210969ef5dbba4d4a3086077ff524de909b_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210969ef5dbba4d4a3086077ff524de909b_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:18,357 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/b5be2827bea94664a45792772bb536eb, store: [table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:18,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/b5be2827bea94664a45792772bb536eb is 175, key is test_row_0/A:col10/1733814917547/Put/seqid=0 2024-12-10T07:15:18,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T07:15:18,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:18,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:18,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:18,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:18,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:18,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:18,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741950_1126 (size=39549) 2024-12-10T07:15:18,408 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/b5be2827bea94664a45792772bb536eb 2024-12-10T07:15:18,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814918372, max timestamp=1733814918410Num uniques 859; 2024-12-10T07:15:18,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814918372, max timestamp=1733814918410Num uniques 859; 2024-12-10T07:15:18,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:15:18,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:15:18,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/7d3030b917724373af48e86c76f150d0 is 50, key is test_row_0/B:col10/1733814917547/Put/seqid=0 2024-12-10T07:15:18,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:18,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814978448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:18,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:18,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814978449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:18,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:18,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:18,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:18,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814978452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:18,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:18,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814978457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814978450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:18,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:18,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741951_1127 (size=12001) 2024-12-10T07:15:18,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/7d3030b917724373af48e86c76f150d0 2024-12-10T07:15:18,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:18,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:18,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/6660049533a64aa8b1b5778dedf06432 is 50, key is test_row_0/C:col10/1733814917547/Put/seqid=0 2024-12-10T07:15:18,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T07:15:18,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:18,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:18,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:18,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:18,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:18,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:18,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741952_1128 (size=12001) 2024-12-10T07:15:18,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/6660049533a64aa8b1b5778dedf06432 2024-12-10T07:15:18,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814978566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:18,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:18,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814978577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:18,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:18,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814978577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:18,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:18,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814978578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:18,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:18,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814978578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:18,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:18,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/b5be2827bea94664a45792772bb536eb as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/b5be2827bea94664a45792772bb536eb 2024-12-10T07:15:18,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/b5be2827bea94664a45792772bb536eb, entries=200, sequenceid=62, filesize=38.6 K 2024-12-10T07:15:18,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/7d3030b917724373af48e86c76f150d0 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/7d3030b917724373af48e86c76f150d0 2024-12-10T07:15:18,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T07:15:18,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/7d3030b917724373af48e86c76f150d0, entries=150, sequenceid=62, filesize=11.7 K 2024-12-10T07:15:18,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:18,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:18,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:18,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:18,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:18,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/6660049533a64aa8b1b5778dedf06432 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/6660049533a64aa8b1b5778dedf06432 2024-12-10T07:15:18,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/6660049533a64aa8b1b5778dedf06432, entries=150, sequenceid=62, filesize=11.7 K 2024-12-10T07:15:18,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=129.71 KB/132820 for 573523ff7c74144a068617629803d4b6 in 432ms, sequenceid=62, compaction requested=true 2024-12-10T07:15:18,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:18,714 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:18,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:15:18,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:18,715 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:18,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:15:18,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:18,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:15:18,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:18,722 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:18,722 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/A is initiating minor compaction (all files) 2024-12-10T07:15:18,722 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/A in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:18,722 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/31e0933d216248ef8cbccd89a28a65e9, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/50dc2eae10534b36bbb1881390225a95, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/b5be2827bea94664a45792772bb536eb] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=107.5 K 2024-12-10T07:15:18,722 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:18,723 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. files: [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/31e0933d216248ef8cbccd89a28a65e9, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/50dc2eae10534b36bbb1881390225a95, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/b5be2827bea94664a45792772bb536eb] 2024-12-10T07:15:18,724 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:18,724 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/B is initiating minor compaction (all files) 2024-12-10T07:15:18,724 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/B in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:18,724 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/4f639267d7a24e9380441020435d4ca0, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/16b23a711b8545c6aa4891c53aa5d204, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/7d3030b917724373af48e86c76f150d0] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=35.2 K 2024-12-10T07:15:18,724 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31e0933d216248ef8cbccd89a28a65e9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733814915992 2024-12-10T07:15:18,725 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f639267d7a24e9380441020435d4ca0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733814915992 2024-12-10T07:15:18,725 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50dc2eae10534b36bbb1881390225a95, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733814916213 2024-12-10T07:15:18,726 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 16b23a711b8545c6aa4891c53aa5d204, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733814916213 2024-12-10T07:15:18,726 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5be2827bea94664a45792772bb536eb, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733814917547 2024-12-10T07:15:18,727 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d3030b917724373af48e86c76f150d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733814917547 2024-12-10T07:15:18,742 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#B#compaction#110 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:18,742 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/e458cac9d6314d73961bfb4553c49690 is 50, key is test_row_0/B:col10/1733814917547/Put/seqid=0 2024-12-10T07:15:18,746 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:18,750 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210b565776a9cda497ea28468252d15a7ba_573523ff7c74144a068617629803d4b6 store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:18,755 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210b565776a9cda497ea28468252d15a7ba_573523ff7c74144a068617629803d4b6, store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:18,755 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210b565776a9cda497ea28468252d15a7ba_573523ff7c74144a068617629803d4b6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:18,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741953_1129 (size=12104) 2024-12-10T07:15:18,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741954_1130 (size=4469) 2024-12-10T07:15:18,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 573523ff7c74144a068617629803d4b6 3/3 column families, dataSize=136.42 KB heapSize=393.47 KB 2024-12-10T07:15:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:18,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=A 2024-12-10T07:15:18,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:18,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=B 2024-12-10T07:15:18,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:18,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=C 2024-12-10T07:15:18,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:18,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814978837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:18,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814978839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:18,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210980cd7d7fdd54a3199595841826c74fb_573523ff7c74144a068617629803d4b6 is 50, key is test_row_0/A:col10/1733814918796/Put/seqid=0 2024-12-10T07:15:18,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:18,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:18,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:18,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T07:15:18,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:18,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:18,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:18,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814978842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:18,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:18,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:18,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814978846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814978846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:18,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:18,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741955_1131 (size=14594) 2024-12-10T07:15:18,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:18,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814978955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:18,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814978956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:18,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:18,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814978965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:18,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:18,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:19,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T07:15:19,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:19,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:19,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:19,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:19,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:19,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:19,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814979165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:19,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:19,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:19,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:19,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:19,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814979176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:19,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:19,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:19,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:19,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:19,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T07:15:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814979176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:19,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:19,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:19,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:19,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:19,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:19,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:19,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:19,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:19,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:19,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814979176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:19,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:19,197 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/e458cac9d6314d73961bfb4553c49690 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/e458cac9d6314d73961bfb4553c49690 2024-12-10T07:15:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814979186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:19,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:19,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:19,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:19,202 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#A#compaction#111 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:19,202 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/026ee71696294d5296102b3e83cf9aa0 is 175, key is test_row_0/A:col10/1733814917547/Put/seqid=0 2024-12-10T07:15:19,207 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 573523ff7c74144a068617629803d4b6/B of 573523ff7c74144a068617629803d4b6 into e458cac9d6314d73961bfb4553c49690(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:19,207 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:19,207 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/B, priority=13, startTime=1733814918715; duration=0sec 2024-12-10T07:15:19,207 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:19,207 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:B 2024-12-10T07:15:19,207 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:19,209 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:19,209 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/C is initiating minor compaction (all files) 2024-12-10T07:15:19,209 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/C in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:19,209 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/71360b8471704734a8a36a1bc822f79c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/af68ca007b374c119f90cb4685a9655e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/6660049533a64aa8b1b5778dedf06432] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=35.2 K 2024-12-10T07:15:19,212 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 71360b8471704734a8a36a1bc822f79c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733814915992 2024-12-10T07:15:19,213 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting af68ca007b374c119f90cb4685a9655e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733814916213 2024-12-10T07:15:19,213 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 6660049533a64aa8b1b5778dedf06432, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733814917547 2024-12-10T07:15:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:19,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:19,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741956_1132 (size=31058) 2024-12-10T07:15:19,271 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#C#compaction#113 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:19,272 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/4a2c71334fa842f3ab7c3aed31032327 is 50, key is test_row_0/C:col10/1733814917547/Put/seqid=0 2024-12-10T07:15:19,293 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/026ee71696294d5296102b3e83cf9aa0 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/026ee71696294d5296102b3e83cf9aa0 2024-12-10T07:15:19,293 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:19,305 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210980cd7d7fdd54a3199595841826c74fb_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210980cd7d7fdd54a3199595841826c74fb_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:19,307 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/4ebb58de2647478c9932e270c7b89556, store: [table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:19,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/4ebb58de2647478c9932e270c7b89556 is 175, key is test_row_0/A:col10/1733814918796/Put/seqid=0 2024-12-10T07:15:19,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741957_1133 (size=12104) 2024-12-10T07:15:19,320 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 573523ff7c74144a068617629803d4b6/A of 573523ff7c74144a068617629803d4b6 into 026ee71696294d5296102b3e83cf9aa0(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:19,320 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:19,320 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/A, priority=13, startTime=1733814918714; duration=0sec 2024-12-10T07:15:19,321 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:19,321 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:A 2024-12-10T07:15:19,327 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/4a2c71334fa842f3ab7c3aed31032327 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/4a2c71334fa842f3ab7c3aed31032327 2024-12-10T07:15:19,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741958_1134 (size=39549) 2024-12-10T07:15:19,329 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/4ebb58de2647478c9932e270c7b89556 2024-12-10T07:15:19,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T07:15:19,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:19,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:19,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:19,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:19,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:19,345 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 573523ff7c74144a068617629803d4b6/C of 573523ff7c74144a068617629803d4b6 into 4a2c71334fa842f3ab7c3aed31032327(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:19,345 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:19,345 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/C, priority=13, startTime=1733814918715; duration=0sec 2024-12-10T07:15:19,345 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:19,345 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:C 2024-12-10T07:15:19,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:19,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/aa3ba355083341af812c630ca221d345 is 50, key is test_row_0/B:col10/1733814918796/Put/seqid=0 2024-12-10T07:15:19,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741959_1135 (size=12001) 2024-12-10T07:15:19,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/aa3ba355083341af812c630ca221d345 2024-12-10T07:15:19,395 INFO [master/b1396b8d2d12:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-10T07:15:19,395 INFO [master/b1396b8d2d12:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-10T07:15:19,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/72b64c4999a145e9921a7cc8ca816f25 is 50, key is test_row_0/C:col10/1733814918796/Put/seqid=0 2024-12-10T07:15:19,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741960_1136 (size=12001) 2024-12-10T07:15:19,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/72b64c4999a145e9921a7cc8ca816f25 2024-12-10T07:15:19,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T07:15:19,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:19,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:19,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:19,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:19,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:19,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:19,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:19,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814979495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:19,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:19,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:19,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:19,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:19,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814979495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:19,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:19,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:19,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:19,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/4ebb58de2647478c9932e270c7b89556 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/4ebb58de2647478c9932e270c7b89556 2024-12-10T07:15:19,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/4ebb58de2647478c9932e270c7b89556, entries=200, sequenceid=95, filesize=38.6 K 2024-12-10T07:15:19,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/aa3ba355083341af812c630ca221d345 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/aa3ba355083341af812c630ca221d345 2024-12-10T07:15:19,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/aa3ba355083341af812c630ca221d345, entries=150, sequenceid=95, filesize=11.7 K 2024-12-10T07:15:19,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:19,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814979505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:19,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/72b64c4999a145e9921a7cc8ca816f25 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/72b64c4999a145e9921a7cc8ca816f25 2024-12-10T07:15:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:19,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/72b64c4999a145e9921a7cc8ca816f25, entries=150, sequenceid=95, filesize=11.7 K 2024-12-10T07:15:19,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~143.13 KB/146560, heapSize ~413.34 KB/423264, currentSize=33.54 KB/34350 for 573523ff7c74144a068617629803d4b6 in 761ms, sequenceid=95, compaction requested=false 2024-12-10T07:15:19,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:19,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-10T07:15:19,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:19,666 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 573523ff7c74144a068617629803d4b6 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T07:15:19,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=A 2024-12-10T07:15:19,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:19,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=B 2024-12-10T07:15:19,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:19,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=C 2024-12-10T07:15:19,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:19,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:19,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:19,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121030d6a3e817794299b0ece62d1300f8e6_573523ff7c74144a068617629803d4b6 is 50, key is test_row_0/A:col10/1733814918817/Put/seqid=0 2024-12-10T07:15:19,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741961_1137 (size=12154) 2024-12-10T07:15:19,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:19,756 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121030d6a3e817794299b0ece62d1300f8e6_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121030d6a3e817794299b0ece62d1300f8e6_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:19,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/726307b9a0a949b89fa754d3ca816921, store: [table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:19,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/726307b9a0a949b89fa754d3ca816921 is 175, key is test_row_0/A:col10/1733814918817/Put/seqid=0 2024-12-10T07:15:19,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741962_1138 (size=30955) 2024-12-10T07:15:19,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733814919743, max timestamp=1733814919807Num uniques 859; 2024-12-10T07:15:19,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T07:15:19,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733814919743, max timestamp=1733814919807Num uniques 859; 2024-12-10T07:15:19,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T07:15:19,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733814919743, max timestamp=1733814919807Num uniques 859; 2024-12-10T07:15:19,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T07:15:19,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733814919910, max timestamp=1733814919994Num uniques 1018; 2024-12-10T07:15:19,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733814919910, max timestamp=1733814919994Num uniques 1018; 2024-12-10T07:15:19,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-10T07:15:19,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-10T07:15:20,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733814919910, max timestamp=1733814919994Num uniques 1018; 2024-12-10T07:15:20,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-10T07:15:20,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:20,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814980073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:20,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:20,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733814920058, max timestamp=1733814920076Num uniques 1077; 2024-12-10T07:15:20,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048  suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-10T07:15:20,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733814920058, max timestamp=1733814920076Num uniques 1077; 2024-12-10T07:15:20,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048  suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-10T07:15:20,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733814920058, max timestamp=1733814920076Num uniques 1077; 2024-12-10T07:15:20,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048  suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-10T07:15:20,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:20,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814980139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:20,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:20,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:20,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814980147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:20,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:20,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814980150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814980152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:20,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:20,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-10T07:15:20,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:20,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814980185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:20,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:20,204 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=106, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/726307b9a0a949b89fa754d3ca816921 2024-12-10T07:15:20,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/95afc3610c7b446bac6fe89ea3c60df6 is 50, key is test_row_0/B:col10/1733814918817/Put/seqid=0 2024-12-10T07:15:20,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814980256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:20,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:20,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741963_1139 (size=12001) 2024-12-10T07:15:20,273 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/95afc3610c7b446bac6fe89ea3c60df6 2024-12-10T07:15:20,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:20,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814980265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814980275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:20,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:20,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814980277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:20,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:20,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/841cf0687b664325b388db317586f8b1 is 50, key is test_row_0/C:col10/1733814918817/Put/seqid=0 2024-12-10T07:15:20,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741964_1140 (size=12001) 2024-12-10T07:15:20,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814980395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:20,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814980465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:20,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:20,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:20,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:20,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814980486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:20,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814980486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:20,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:20,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814980495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:20,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:20,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:20,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814980715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:20,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:20,778 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/841cf0687b664325b388db317586f8b1 2024-12-10T07:15:20,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/726307b9a0a949b89fa754d3ca816921 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/726307b9a0a949b89fa754d3ca816921 2024-12-10T07:15:20,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:20,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814980784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:20,794 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/726307b9a0a949b89fa754d3ca816921, entries=150, sequenceid=106, filesize=30.2 K 2024-12-10T07:15:20,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/95afc3610c7b446bac6fe89ea3c60df6 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/95afc3610c7b446bac6fe89ea3c60df6 2024-12-10T07:15:20,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:20,814 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/95afc3610c7b446bac6fe89ea3c60df6, entries=150, sequenceid=106, filesize=11.7 K 2024-12-10T07:15:20,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:20,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814980805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:20,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:20,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/841cf0687b664325b388db317586f8b1 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/841cf0687b664325b388db317586f8b1 2024-12-10T07:15:20,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:20,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814980815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:20,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:20,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:20,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814980815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:20,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:20,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:20,861 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/841cf0687b664325b388db317586f8b1, entries=150, sequenceid=106, filesize=11.7 K 2024-12-10T07:15:20,863 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 573523ff7c74144a068617629803d4b6 in 1197ms, sequenceid=106, compaction requested=true 2024-12-10T07:15:20,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:20,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:20,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-12-10T07:15:20,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-12-10T07:15:20,871 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-10T07:15:20,871 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.8660 sec 2024-12-10T07:15:20,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 4.8810 sec 2024-12-10T07:15:21,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733814920138, max timestamp=1733814921241Num uniques 1136; 2024-12-10T07:15:21,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584  suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-10T07:15:21,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733814920138, max timestamp=1733814921241Num uniques 1136; 2024-12-10T07:15:21,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584  suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-10T07:15:21,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733814920138, max timestamp=1733814921241Num uniques 1136; 2024-12-10T07:15:21,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584  suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-10T07:15:21,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 573523ff7c74144a068617629803d4b6 3/3 column families, dataSize=53.67 KB heapSize=143.39 KB 2024-12-10T07:15:21,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=A 2024-12-10T07:15:21,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:21,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=B 2024-12-10T07:15:21,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:21,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=C 2024-12-10T07:15:21,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:21,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:21,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210ccbc0d625c234f25b53d7986d876f767_573523ff7c74144a068617629803d4b6 is 50, key is test_row_0/A:col10/1733814921270/Put/seqid=0 2024-12-10T07:15:21,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741965_1141 (size=14794) 2024-12-10T07:15:21,369 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:21,383 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210ccbc0d625c234f25b53d7986d876f767_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ccbc0d625c234f25b53d7986d876f767_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:21,387 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/98d123de75304bab8dd88ac359d5e0c2, store: [table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:21,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/98d123de75304bab8dd88ac359d5e0c2 is 175, key is test_row_0/A:col10/1733814921270/Put/seqid=0 2024-12-10T07:15:21,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814921402, max timestamp=1733814921430Num uniques 859; 2024-12-10T07:15:21,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814921402, max timestamp=1733814921430Num uniques 859; 2024-12-10T07:15:21,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:15:21,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:15:21,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741966_1142 (size=39749) 2024-12-10T07:15:21,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733814921402, max timestamp=1733814921430Num uniques 859; 2024-12-10T07:15:21,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T07:15:21,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814981514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:21,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814981519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:21,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814981525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814981525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:21,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814981526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:21,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:21,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:21,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814981645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:21,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:21,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814981645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:21,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:21,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:21,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814981648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:21,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814981648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:21,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:21,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:21,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814981654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:21,837 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/98d123de75304bab8dd88ac359d5e0c2 2024-12-10T07:15:21,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814981876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:21,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814981876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:21,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:21,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814981876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:21,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814981876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:21,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814981876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:21,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/c3a759b156394c8ca3ae6bde57dd9fc9 is 50, key is test_row_0/B:col10/1733814921270/Put/seqid=0 2024-12-10T07:15:21,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741967_1143 (size=12151) 2024-12-10T07:15:21,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/c3a759b156394c8ca3ae6bde57dd9fc9 2024-12-10T07:15:22,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/e39cd111f67c4d098a6f152312d43cbb is 50, key is test_row_0/C:col10/1733814921270/Put/seqid=0 2024-12-10T07:15:22,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741968_1144 (size=12151) 2024-12-10T07:15:22,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/e39cd111f67c4d098a6f152312d43cbb 2024-12-10T07:15:22,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/98d123de75304bab8dd88ac359d5e0c2 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/98d123de75304bab8dd88ac359d5e0c2 2024-12-10T07:15:22,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/98d123de75304bab8dd88ac359d5e0c2, entries=200, sequenceid=182, filesize=38.8 K 2024-12-10T07:15:22,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/c3a759b156394c8ca3ae6bde57dd9fc9 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/c3a759b156394c8ca3ae6bde57dd9fc9 2024-12-10T07:15:22,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/c3a759b156394c8ca3ae6bde57dd9fc9, entries=150, sequenceid=182, filesize=11.9 K 2024-12-10T07:15:22,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/e39cd111f67c4d098a6f152312d43cbb as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/e39cd111f67c4d098a6f152312d43cbb 2024-12-10T07:15:22,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/e39cd111f67c4d098a6f152312d43cbb, entries=150, sequenceid=182, filesize=11.9 K 2024-12-10T07:15:22,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~143.34 KB/146784, currentSize=138.65 KB/141980 for 573523ff7c74144a068617629803d4b6 in 935ms, sequenceid=182, compaction requested=true 2024-12-10T07:15:22,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:22,214 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:15:22,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:15:22,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:22,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:15:22,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:22,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:15:22,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T07:15:22,217 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:15:22,217 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141311 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:15:22,217 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/A is initiating minor compaction (all files) 2024-12-10T07:15:22,217 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/A in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:22,217 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/026ee71696294d5296102b3e83cf9aa0, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/4ebb58de2647478c9932e270c7b89556, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/726307b9a0a949b89fa754d3ca816921, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/98d123de75304bab8dd88ac359d5e0c2] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=138.0 K 2024-12-10T07:15:22,217 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:22,218 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. files: [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/026ee71696294d5296102b3e83cf9aa0, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/4ebb58de2647478c9932e270c7b89556, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/726307b9a0a949b89fa754d3ca816921, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/98d123de75304bab8dd88ac359d5e0c2] 2024-12-10T07:15:22,218 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 026ee71696294d5296102b3e83cf9aa0, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733814917547 2024-12-10T07:15:22,219 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ebb58de2647478c9932e270c7b89556, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733814918430 2024-12-10T07:15:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:22,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733814921479, max timestamp=1733814922218Num uniques 1068; 2024-12-10T07:15:22,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733814921479, max timestamp=1733814922218Num uniques 1068; 2024-12-10T07:15:22,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-10T07:15:22,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-10T07:15:22,228 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 726307b9a0a949b89fa754d3ca816921, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1733814918817 2024-12-10T07:15:22,229 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 98d123de75304bab8dd88ac359d5e0c2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733814921241 2024-12-10T07:15:22,229 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:15:22,229 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/B is initiating minor compaction (all files) 2024-12-10T07:15:22,229 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/B in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:22,229 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/e458cac9d6314d73961bfb4553c49690, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/aa3ba355083341af812c630ca221d345, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/95afc3610c7b446bac6fe89ea3c60df6, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/c3a759b156394c8ca3ae6bde57dd9fc9] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=47.1 K 2024-12-10T07:15:22,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733814921479, max timestamp=1733814922218Num uniques 1018; 2024-12-10T07:15:22,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T07:15:22,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 573523ff7c74144a068617629803d4b6 3/3 column families, dataSize=53.67 KB heapSize=145.08 KB 2024-12-10T07:15:22,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=A 2024-12-10T07:15:22,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:22,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=B 2024-12-10T07:15:22,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:22,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=C 2024-12-10T07:15:22,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:22,236 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting e458cac9d6314d73961bfb4553c49690, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733814917547 2024-12-10T07:15:22,245 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa3ba355083341af812c630ca221d345, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733814918430 2024-12-10T07:15:22,254 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95afc3610c7b446bac6fe89ea3c60df6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1733814918817 2024-12-10T07:15:22,262 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3a759b156394c8ca3ae6bde57dd9fc9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733814921263 2024-12-10T07:15:22,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210ddf7022632c34588a31da5b85b5cef27_573523ff7c74144a068617629803d4b6 is 50, key is test_row_0/A:col10/1733814922225/Put/seqid=0 2024-12-10T07:15:22,329 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:22,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733814922295, max timestamp=1733814922330Num uniques 859; 2024-12-10T07:15:22,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-10T07:15:22,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733814922295, max timestamp=1733814922330Num uniques 859; 2024-12-10T07:15:22,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-10T07:15:22,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733814922295, max timestamp=1733814922330Num uniques 859; 2024-12-10T07:15:22,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-10T07:15:22,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741969_1145 (size=12304) 2024-12-10T07:15:22,365 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:22,371 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#B#compaction#124 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:22,372 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/b191e2d24d814d64a58899c277b4b604 is 50, key is test_row_0/B:col10/1733814921270/Put/seqid=0 2024-12-10T07:15:22,372 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210ddf7022632c34588a31da5b85b5cef27_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ddf7022632c34588a31da5b85b5cef27_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:22,374 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/2df345b96c7d464f9698e97165d72840, store: [table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:22,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/2df345b96c7d464f9698e97165d72840 is 175, key is test_row_0/A:col10/1733814922225/Put/seqid=0 2024-12-10T07:15:22,378 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210a478dc6f3e7d41c7a2554a70a19579f9_573523ff7c74144a068617629803d4b6 store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:22,382 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210a478dc6f3e7d41c7a2554a70a19579f9_573523ff7c74144a068617629803d4b6, store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:22,382 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a478dc6f3e7d41c7a2554a70a19579f9_573523ff7c74144a068617629803d4b6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:22,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:22,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814982409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:22,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814982406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:22,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:22,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814982421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:22,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:22,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814982422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:22,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:22,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814982422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:22,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:22,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741970_1146 (size=12391) 2024-12-10T07:15:22,460 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/b191e2d24d814d64a58899c277b4b604 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/b191e2d24d814d64a58899c277b4b604 2024-12-10T07:15:22,468 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 573523ff7c74144a068617629803d4b6/B of 573523ff7c74144a068617629803d4b6 into b191e2d24d814d64a58899c277b4b604(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:22,468 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:22,468 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/B, priority=12, startTime=1733814922215; duration=0sec 2024-12-10T07:15:22,468 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:22,470 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:B 2024-12-10T07:15:22,470 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:15:22,472 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:15:22,473 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/C is initiating minor compaction (all files) 2024-12-10T07:15:22,473 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/C in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:22,473 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/4a2c71334fa842f3ab7c3aed31032327, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/72b64c4999a145e9921a7cc8ca816f25, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/841cf0687b664325b388db317586f8b1, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/e39cd111f67c4d098a6f152312d43cbb] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=47.1 K 2024-12-10T07:15:22,474 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a2c71334fa842f3ab7c3aed31032327, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733814917547 2024-12-10T07:15:22,474 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72b64c4999a145e9921a7cc8ca816f25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733814918430 2024-12-10T07:15:22,475 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 841cf0687b664325b388db317586f8b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1733814918817 2024-12-10T07:15:22,475 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting e39cd111f67c4d098a6f152312d43cbb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733814921263 2024-12-10T07:15:22,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741971_1147 (size=31105) 2024-12-10T07:15:22,493 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/2df345b96c7d464f9698e97165d72840 2024-12-10T07:15:22,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741972_1148 (size=4469) 2024-12-10T07:15:22,506 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#C#compaction#125 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:22,507 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/4bd41cd1d6b642a5be64938a60df95c2 is 50, key is test_row_0/C:col10/1733814921270/Put/seqid=0 2024-12-10T07:15:22,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/31f2a2244ad94369a657038f285b37ba is 50, key is test_row_0/B:col10/1733814922225/Put/seqid=0 2024-12-10T07:15:22,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741973_1149 (size=12391) 2024-12-10T07:15:22,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814982535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:22,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814982536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:22,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:22,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:22,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814982545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:22,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814982545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:22,551 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/4bd41cd1d6b642a5be64938a60df95c2 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/4bd41cd1d6b642a5be64938a60df95c2 2024-12-10T07:15:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:22,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814982545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,567 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 573523ff7c74144a068617629803d4b6/C of 573523ff7c74144a068617629803d4b6 into 4bd41cd1d6b642a5be64938a60df95c2(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:22,567 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:22,567 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/C, priority=12, startTime=1733814922215; duration=0sec 2024-12-10T07:15:22,567 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:22,567 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:C 2024-12-10T07:15:22,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741974_1150 (size=12151) 2024-12-10T07:15:22,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/31f2a2244ad94369a657038f285b37ba 2024-12-10T07:15:22,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/42468d3db4c14c1da24ccad3fd435f95 is 50, key is test_row_0/C:col10/1733814922225/Put/seqid=0 2024-12-10T07:15:22,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741975_1151 (size=12151) 2024-12-10T07:15:22,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/42468d3db4c14c1da24ccad3fd435f95 2024-12-10T07:15:22,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/2df345b96c7d464f9698e97165d72840 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/2df345b96c7d464f9698e97165d72840 2024-12-10T07:15:22,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/2df345b96c7d464f9698e97165d72840, entries=150, sequenceid=224, filesize=30.4 K 2024-12-10T07:15:22,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/31f2a2244ad94369a657038f285b37ba as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/31f2a2244ad94369a657038f285b37ba 2024-12-10T07:15:22,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/31f2a2244ad94369a657038f285b37ba, entries=150, sequenceid=224, filesize=11.9 K 2024-12-10T07:15:22,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/42468d3db4c14c1da24ccad3fd435f95 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/42468d3db4c14c1da24ccad3fd435f95 2024-12-10T07:15:22,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/42468d3db4c14c1da24ccad3fd435f95, entries=150, sequenceid=224, filesize=11.9 K 2024-12-10T07:15:22,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.03 KB/148512, currentSize=140.89 KB/144270 for 573523ff7c74144a068617629803d4b6 in 515ms, sequenceid=224, compaction requested=false 2024-12-10T07:15:22,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:22,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 573523ff7c74144a068617629803d4b6 3/3 column families, dataSize=154.31 KB heapSize=426 KB 2024-12-10T07:15:22,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=A 2024-12-10T07:15:22,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:22,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=B 2024-12-10T07:15:22,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:22,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=C 2024-12-10T07:15:22,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:22,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814982774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:22,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814982777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:22,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412103d949a8db9ca44de9a7a1e6eaed3ef95_573523ff7c74144a068617629803d4b6 is 50, key is test_row_0/A:col10/1733814922765/Put/seqid=0 2024-12-10T07:15:22,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:22,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814982780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:22,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814982781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:22,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:22,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814982781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:22,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741976_1152 (size=19974) 2024-12-10T07:15:22,856 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:22,863 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412103d949a8db9ca44de9a7a1e6eaed3ef95_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103d949a8db9ca44de9a7a1e6eaed3ef95_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:22,865 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/fbfe75bf23a149ebb94a11586d8e1637, store: [table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:22,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/fbfe75bf23a149ebb94a11586d8e1637 is 175, key is test_row_0/A:col10/1733814922765/Put/seqid=0 2024-12-10T07:15:22,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814982897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:22,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,902 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:22,914 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#A#compaction#123 average throughput is 0.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:22,914 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/186da5696d7f4a80b7e46bede4321e12 is 175, key is test_row_0/A:col10/1733814921270/Put/seqid=0 2024-12-10T07:15:22,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814982909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:22,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:22,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814982909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:22,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:22,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:22,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741977_1153 (size=57233) 2024-12-10T07:15:22,930 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/fbfe75bf23a149ebb94a11586d8e1637 2024-12-10T07:15:22,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814982909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:22,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741978_1154 (size=31345) 2024-12-10T07:15:22,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/1f4f8990b10f491b9dbfeb9678923168 is 50, key is test_row_0/B:col10/1733814922765/Put/seqid=0 2024-12-10T07:15:22,960 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/186da5696d7f4a80b7e46bede4321e12 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/186da5696d7f4a80b7e46bede4321e12 2024-12-10T07:15:22,975 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 573523ff7c74144a068617629803d4b6/A of 573523ff7c74144a068617629803d4b6 into 186da5696d7f4a80b7e46bede4321e12(size=30.6 K), total size for store is 61.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:22,975 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:22,975 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/A, priority=12, startTime=1733814922214; duration=0sec 2024-12-10T07:15:22,976 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:22,976 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:A 2024-12-10T07:15:23,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741979_1155 (size=12301) 2024-12-10T07:15:23,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/1f4f8990b10f491b9dbfeb9678923168 2024-12-10T07:15:23,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/d75b15992ea441a1ba487ef58afc33f0 is 50, key is test_row_0/C:col10/1733814922765/Put/seqid=0 2024-12-10T07:15:23,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814983095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:23,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:23,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:23,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:23,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814983104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:23,108 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:23,108 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:23,108 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:23,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741980_1156 (size=12301) 2024-12-10T07:15:23,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:23,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814983125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:23,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814983125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:23,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:23,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:23,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:23,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:23,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814983147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:23,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:23,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:23,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:23,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814983415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:23,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:23,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:23,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:23,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814983434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:23,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814983445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:23,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:23,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814983465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:23,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:23,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:23,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:23,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/d75b15992ea441a1ba487ef58afc33f0 2024-12-10T07:15:23,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/fbfe75bf23a149ebb94a11586d8e1637 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/fbfe75bf23a149ebb94a11586d8e1637 2024-12-10T07:15:23,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/fbfe75bf23a149ebb94a11586d8e1637, entries=300, sequenceid=263, filesize=55.9 K 2024-12-10T07:15:23,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/1f4f8990b10f491b9dbfeb9678923168 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/1f4f8990b10f491b9dbfeb9678923168 2024-12-10T07:15:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/1f4f8990b10f491b9dbfeb9678923168, entries=150, sequenceid=263, filesize=12.0 K 2024-12-10T07:15:23,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/d75b15992ea441a1ba487ef58afc33f0 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/d75b15992ea441a1ba487ef58afc33f0 2024-12-10T07:15:23,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/d75b15992ea441a1ba487ef58afc33f0, entries=150, sequenceid=263, filesize=12.0 K 2024-12-10T07:15:23,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~445.88 KB/456576, currentSize=26.84 KB/27480 for 573523ff7c74144a068617629803d4b6 in 849ms, sequenceid=263, compaction requested=true 2024-12-10T07:15:23,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:23,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:15:23,615 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:23,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:23,616 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:23,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:15:23,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:23,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:15:23,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:23,618 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:23,619 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/A is initiating minor compaction (all files) 2024-12-10T07:15:23,619 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/A in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:23,619 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/186da5696d7f4a80b7e46bede4321e12, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/2df345b96c7d464f9698e97165d72840, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/fbfe75bf23a149ebb94a11586d8e1637] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=116.9 K 2024-12-10T07:15:23,619 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:23,619 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. files: [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/186da5696d7f4a80b7e46bede4321e12, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/2df345b96c7d464f9698e97165d72840, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/fbfe75bf23a149ebb94a11586d8e1637] 2024-12-10T07:15:23,619 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36843 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:23,619 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/B is initiating minor compaction (all files) 2024-12-10T07:15:23,619 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/B in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:23,619 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/b191e2d24d814d64a58899c277b4b604, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/31f2a2244ad94369a657038f285b37ba, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/1f4f8990b10f491b9dbfeb9678923168] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=36.0 K 2024-12-10T07:15:23,620 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 186da5696d7f4a80b7e46bede4321e12, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733814921263 2024-12-10T07:15:23,620 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting b191e2d24d814d64a58899c277b4b604, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733814921263 2024-12-10T07:15:23,621 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 31f2a2244ad94369a657038f285b37ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733814922222 2024-12-10T07:15:23,621 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2df345b96c7d464f9698e97165d72840, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733814922222 2024-12-10T07:15:23,622 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f4f8990b10f491b9dbfeb9678923168, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733814922419 2024-12-10T07:15:23,623 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbfe75bf23a149ebb94a11586d8e1637, keycount=300, bloomtype=ROW, size=55.9 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733814922398 2024-12-10T07:15:23,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,648 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#B#compaction#131 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:23,649 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/5bc2beaa1cae48f29cfd8b5e6c6b8fd6 is 50, key is test_row_0/B:col10/1733814922765/Put/seqid=0 2024-12-10T07:15:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,662 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,682 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210081eea9f6f284b529ca286cb77bc6b64_573523ff7c74144a068617629803d4b6 store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,690 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210081eea9f6f284b529ca286cb77bc6b64_573523ff7c74144a068617629803d4b6, store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:23,690 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210081eea9f6f284b529ca286cb77bc6b64_573523ff7c74144a068617629803d4b6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:23,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741981_1157 (size=12643) 2024-12-10T07:15:23,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,707 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/5bc2beaa1cae48f29cfd8b5e6c6b8fd6 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/5bc2beaa1cae48f29cfd8b5e6c6b8fd6 2024-12-10T07:15:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,720 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 573523ff7c74144a068617629803d4b6/B of 573523ff7c74144a068617629803d4b6 into 5bc2beaa1cae48f29cfd8b5e6c6b8fd6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:23,720 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:23,720 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/B, priority=13, startTime=1733814923615; duration=0sec 2024-12-10T07:15:23,720 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:23,720 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:B 2024-12-10T07:15:23,720 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:23,723 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36843 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:23,723 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/C is initiating minor compaction (all files) 2024-12-10T07:15:23,724 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/C in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:23,724 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/4bd41cd1d6b642a5be64938a60df95c2, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/42468d3db4c14c1da24ccad3fd435f95, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/d75b15992ea441a1ba487ef58afc33f0] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=36.0 K 2024-12-10T07:15:23,724 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bd41cd1d6b642a5be64938a60df95c2, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733814921263 2024-12-10T07:15:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,725 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 42468d3db4c14c1da24ccad3fd435f95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733814922222 2024-12-10T07:15:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,728 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting d75b15992ea441a1ba487ef58afc33f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733814922419 2024-12-10T07:15:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741982_1158 (size=4469) 2024-12-10T07:15:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,751 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#A#compaction#132 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:23,751 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/bf846d28d18f46799b35859122ff3379 is 175, key is test_row_0/A:col10/1733814922765/Put/seqid=0 2024-12-10T07:15:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,763 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#C#compaction#133 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,763 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/ec4ec1fda08a4fdba8747fe10bf91f54 is 50, key is test_row_0/C:col10/1733814922765/Put/seqid=0 2024-12-10T07:15:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741983_1159 (size=31597) 2024-12-10T07:15:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741984_1160 (size=12643) 2024-12-10T07:15:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,820 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/ec4ec1fda08a4fdba8747fe10bf91f54 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/ec4ec1fda08a4fdba8747fe10bf91f54 2024-12-10T07:15:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,831 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 573523ff7c74144a068617629803d4b6/C of 573523ff7c74144a068617629803d4b6 into ec4ec1fda08a4fdba8747fe10bf91f54(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:23,831 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:23,831 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/C, priority=13, startTime=1733814923616; duration=0sec 2024-12-10T07:15:23,831 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:23,831 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:C 2024-12-10T07:15:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:24,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 573523ff7c74144a068617629803d4b6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T07:15:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:24,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=A 2024-12-10T07:15:24,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:24,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=B 2024-12-10T07:15:24,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:24,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=C 2024-12-10T07:15:24,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:24,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210ef5098068af2448fa2687e4cf3b1622a_573523ff7c74144a068617629803d4b6 is 50, key is test_row_1/A:col10/1733814924011/Put/seqid=0 2024-12-10T07:15:24,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733814924073, max timestamp=1733814924100Num uniques 859; 2024-12-10T07:15:24,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T07:15:24,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741985_1161 (size=14994) 2024-12-10T07:15:24,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733814924073, max timestamp=1733814924100Num uniques 859; 2024-12-10T07:15:24,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T07:15:24,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733814924073, max timestamp=1733814924100Num uniques 859; 2024-12-10T07:15:24,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T07:15:24,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:24,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814984153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:24,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814984165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:24,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:24,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814984168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:24,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:24,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814984173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:24,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:24,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814984174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:24,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:24,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:24,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:24,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:24,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-10T07:15:24,225 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:15:24,226 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:15:24,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-10T07:15:24,233 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:15:24,234 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:15:24,234 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:15:24,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T07:15:24,249 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/bf846d28d18f46799b35859122ff3379 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/bf846d28d18f46799b35859122ff3379 2024-12-10T07:15:24,259 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 573523ff7c74144a068617629803d4b6/A of 573523ff7c74144a068617629803d4b6 into bf846d28d18f46799b35859122ff3379(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:24,259 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:24,259 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/A, priority=13, startTime=1733814923615; duration=0sec 2024-12-10T07:15:24,259 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:24,259 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:A 2024-12-10T07:15:24,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:24,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814984287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:24,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:24,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:24,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814984295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:24,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:24,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814984295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:24,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:24,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814984296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:24,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:24,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:24,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:24,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:24,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814984305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:24,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T07:15:24,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T07:15:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:24,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:24,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:24,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:24,510 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:24,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814984509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:24,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814984517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:24,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:24,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814984517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:24,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814984517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:24,523 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210ef5098068af2448fa2687e4cf3b1622a_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ef5098068af2448fa2687e4cf3b1622a_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:24,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:24,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:24,529 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/ffaf96053ddd49e3b7c70305ed4dafa1, store: [table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:24,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/ffaf96053ddd49e3b7c70305ed4dafa1 is 175, key is test_row_1/A:col10/1733814924011/Put/seqid=0 2024-12-10T07:15:24,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:24,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814984524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:24,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:24,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T07:15:24,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:24,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:24,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:24,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:24,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T07:15:24,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741986_1162 (size=39945) 2024-12-10T07:15:24,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T07:15:24,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:24,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:24,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:24,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:24,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:24,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814984825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:24,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814984825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:24,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:24,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:24,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814984834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814984834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:24,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814984845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:24,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:24,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T07:15:24,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T07:15:24,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:24,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:24,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:24,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:24,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:25,032 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/ffaf96053ddd49e3b7c70305ed4dafa1 2024-12-10T07:15:25,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T07:15:25,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:25,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:25,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:25,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:25,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:25,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:25,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/f24922ff033249f183b7193cc54e68ff is 50, key is test_row_1/B:col10/1733814924011/Put/seqid=0 2024-12-10T07:15:25,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741987_1163 (size=9857) 2024-12-10T07:15:25,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/f24922ff033249f183b7193cc54e68ff 2024-12-10T07:15:25,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/e8ed2676de8b4e46a336b16674b382bb is 50, key is test_row_1/C:col10/1733814924011/Put/seqid=0 2024-12-10T07:15:25,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741988_1164 (size=9857) 2024-12-10T07:15:25,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/e8ed2676de8b4e46a336b16674b382bb 2024-12-10T07:15:25,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/ffaf96053ddd49e3b7c70305ed4dafa1 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/ffaf96053ddd49e3b7c70305ed4dafa1 2024-12-10T07:15:25,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/ffaf96053ddd49e3b7c70305ed4dafa1, entries=200, sequenceid=277, filesize=39.0 K 2024-12-10T07:15:25,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/f24922ff033249f183b7193cc54e68ff as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/f24922ff033249f183b7193cc54e68ff 2024-12-10T07:15:25,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/f24922ff033249f183b7193cc54e68ff, entries=100, sequenceid=277, filesize=9.6 K 2024-12-10T07:15:25,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/e8ed2676de8b4e46a336b16674b382bb as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/e8ed2676de8b4e46a336b16674b382bb 2024-12-10T07:15:25,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T07:15:25,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:25,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:25,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:25,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:25,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:25,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:25,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/e8ed2676de8b4e46a336b16674b382bb, entries=100, sequenceid=277, filesize=9.6 K 2024-12-10T07:15:25,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for 573523ff7c74144a068617629803d4b6 in 1197ms, sequenceid=277, compaction requested=false 2024-12-10T07:15:25,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:25,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 573523ff7c74144a068617629803d4b6 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-12-10T07:15:25,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=A 2024-12-10T07:15:25,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T07:15:25,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=B 2024-12-10T07:15:25,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T07:15:25,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=C 2024-12-10T07:15:25,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T07:15:25,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:25,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T07:15:25,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:25,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:25,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:25,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:25,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:25,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T07:15:25,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210306d03b98d0f495a8df5d10aeeac4db9_573523ff7c74144a068617629803d4b6 is 50, key is test_row_0/A:col10/1733814924156/Put/seqid=0 2024-12-10T07:15:25,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:25,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814985369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:25,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:25,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814985385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:25,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:25,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814985386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:25,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:25,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:25,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814985388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:25,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:25,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:25,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:25,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814985409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:25,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:25,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741989_1165 (size=12454) 2024-12-10T07:15:25,461 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:25,471 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210306d03b98d0f495a8df5d10aeeac4db9_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210306d03b98d0f495a8df5d10aeeac4db9_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:25,473 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/884d943af6f943319c49b3baccc311fc, store: [table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:25,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/884d943af6f943319c49b3baccc311fc is 175, key is test_row_0/A:col10/1733814924156/Put/seqid=0 2024-12-10T07:15:25,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:25,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814985495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:25,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:25,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741990_1166 (size=31255) 2024-12-10T07:15:25,511 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/884d943af6f943319c49b3baccc311fc 2024-12-10T07:15:25,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:25,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814985504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:25,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:25,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:25,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814985504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:25,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T07:15:25,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:25,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:25,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:25,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:25,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:25,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:25,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:25,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814985515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:25,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:25,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:25,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814985517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:25,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/41eb4ec95803460cac59824b7c57aa79 is 50, key is test_row_0/B:col10/1733814924156/Put/seqid=0 2024-12-10T07:15:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:25,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741991_1167 (size=12301) 2024-12-10T07:15:25,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/41eb4ec95803460cac59824b7c57aa79 2024-12-10T07:15:25,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/97b53732d69e4269b6577ad6ff150de3 is 50, key is test_row_0/C:col10/1733814924156/Put/seqid=0 2024-12-10T07:15:25,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741992_1168 (size=12301) 2024-12-10T07:15:25,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T07:15:25,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:25,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:25,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:25,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:25,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:25,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:25,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814985705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:25,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:25,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:25,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814985715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:25,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:25,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:25,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814985729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:25,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:25,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814985737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:25,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814985737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:25,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:25,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:25,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:25,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T07:15:25,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:25,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:25,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:25,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:25,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:25,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:26,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T07:15:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:26,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:26,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:26,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814986025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:26,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814986037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:26,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:26,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/97b53732d69e4269b6577ad6ff150de3 2024-12-10T07:15:26,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814986044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:26,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:26,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:26,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814986056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:26,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:26,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:26,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814986056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:26,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/884d943af6f943319c49b3baccc311fc as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/884d943af6f943319c49b3baccc311fc 2024-12-10T07:15:26,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/884d943af6f943319c49b3baccc311fc, entries=150, sequenceid=316, filesize=30.5 K 2024-12-10T07:15:26,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/41eb4ec95803460cac59824b7c57aa79 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/41eb4ec95803460cac59824b7c57aa79 2024-12-10T07:15:26,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/41eb4ec95803460cac59824b7c57aa79, entries=150, sequenceid=316, filesize=12.0 K 2024-12-10T07:15:26,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/97b53732d69e4269b6577ad6ff150de3 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/97b53732d69e4269b6577ad6ff150de3 2024-12-10T07:15:26,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T07:15:26,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:26,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:26,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:26,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:26,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:26,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/97b53732d69e4269b6577ad6ff150de3, entries=150, sequenceid=316, filesize=12.0 K 2024-12-10T07:15:26,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~436.03 KB/446496, currentSize=33.54 KB/34350 for 573523ff7c74144a068617629803d4b6 in 811ms, sequenceid=316, compaction requested=true 2024-12-10T07:15:26,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:26,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:15:26,169 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:26,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:26,169 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:26,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:15:26,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:26,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:15:26,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:26,189 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:26,189 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/A is initiating minor compaction (all files) 2024-12-10T07:15:26,189 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/A in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:26,189 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/bf846d28d18f46799b35859122ff3379, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/ffaf96053ddd49e3b7c70305ed4dafa1, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/884d943af6f943319c49b3baccc311fc] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=100.4 K 2024-12-10T07:15:26,189 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:26,189 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. files: [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/bf846d28d18f46799b35859122ff3379, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/ffaf96053ddd49e3b7c70305ed4dafa1, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/884d943af6f943319c49b3baccc311fc] 2024-12-10T07:15:26,192 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34801 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:26,192 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/B is initiating minor compaction (all files) 2024-12-10T07:15:26,192 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/B in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:26,193 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf846d28d18f46799b35859122ff3379, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733814922419 2024-12-10T07:15:26,193 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/5bc2beaa1cae48f29cfd8b5e6c6b8fd6, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/f24922ff033249f183b7193cc54e68ff, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/41eb4ec95803460cac59824b7c57aa79] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=34.0 K 2024-12-10T07:15:26,194 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffaf96053ddd49e3b7c70305ed4dafa1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733814922778 2024-12-10T07:15:26,195 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bc2beaa1cae48f29cfd8b5e6c6b8fd6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733814922419 2024-12-10T07:15:26,205 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 884d943af6f943319c49b3baccc311fc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733814924156 2024-12-10T07:15:26,209 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting f24922ff033249f183b7193cc54e68ff, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733814924011 2024-12-10T07:15:26,212 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 41eb4ec95803460cac59824b7c57aa79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733814924156 2024-12-10T07:15:26,274 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:26,283 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#B#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:26,284 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/fd9ed32eb9c04dc098c9bcbc89a3de2f is 50, key is test_row_0/B:col10/1733814924156/Put/seqid=0 2024-12-10T07:15:26,305 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210ddf174d7739540879da777262141249e_573523ff7c74144a068617629803d4b6 store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:26,307 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210ddf174d7739540879da777262141249e_573523ff7c74144a068617629803d4b6, store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:26,307 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210ddf174d7739540879da777262141249e_573523ff7c74144a068617629803d4b6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:26,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-10T07:15:26,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:26,321 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 573523ff7c74144a068617629803d4b6 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T07:15:26,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=A 2024-12-10T07:15:26,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:26,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=B 2024-12-10T07:15:26,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:26,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=C 2024-12-10T07:15:26,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:26,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741993_1169 (size=12745) 2024-12-10T07:15:26,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741994_1170 (size=4469) 2024-12-10T07:15:26,371 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/fd9ed32eb9c04dc098c9bcbc89a3de2f as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/fd9ed32eb9c04dc098c9bcbc89a3de2f 2024-12-10T07:15:26,373 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#A#compaction#140 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:26,374 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/08ad69f7ebd14dd1aba06d268acb548c is 175, key is test_row_0/A:col10/1733814924156/Put/seqid=0 2024-12-10T07:15:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T07:15:26,386 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 573523ff7c74144a068617629803d4b6/B of 573523ff7c74144a068617629803d4b6 into fd9ed32eb9c04dc098c9bcbc89a3de2f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:26,386 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:26,386 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/B, priority=13, startTime=1733814926169; duration=0sec 2024-12-10T07:15:26,386 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:26,386 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:B 2024-12-10T07:15:26,386 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:26,393 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34801 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:26,394 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/C is initiating minor compaction (all files) 2024-12-10T07:15:26,394 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/C in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:26,394 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/ec4ec1fda08a4fdba8747fe10bf91f54, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/e8ed2676de8b4e46a336b16674b382bb, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/97b53732d69e4269b6577ad6ff150de3] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=34.0 K 2024-12-10T07:15:26,397 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting ec4ec1fda08a4fdba8747fe10bf91f54, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733814922419 2024-12-10T07:15:26,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121037caa257f8064e2999072db0f715ce31_573523ff7c74144a068617629803d4b6 is 50, key is test_row_0/A:col10/1733814925378/Put/seqid=0 2024-12-10T07:15:26,403 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting e8ed2676de8b4e46a336b16674b382bb, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1733814924011 2024-12-10T07:15:26,409 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 97b53732d69e4269b6577ad6ff150de3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733814924156 2024-12-10T07:15:26,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741995_1171 (size=31699) 2024-12-10T07:15:26,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741996_1172 (size=12454) 2024-12-10T07:15:26,452 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/08ad69f7ebd14dd1aba06d268acb548c as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/08ad69f7ebd14dd1aba06d268acb548c 2024-12-10T07:15:26,468 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#C#compaction#143 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:26,469 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/72c7305da8ed424394b9b72b16f77781 is 50, key is test_row_0/C:col10/1733814924156/Put/seqid=0 2024-12-10T07:15:26,498 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 573523ff7c74144a068617629803d4b6/A of 573523ff7c74144a068617629803d4b6 into 08ad69f7ebd14dd1aba06d268acb548c(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:26,498 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:26,498 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/A, priority=13, startTime=1733814926169; duration=0sec 2024-12-10T07:15:26,498 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:26,499 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:A 2024-12-10T07:15:26,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741997_1173 (size=12745) 2024-12-10T07:15:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:26,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:26,578 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/72c7305da8ed424394b9b72b16f77781 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/72c7305da8ed424394b9b72b16f77781 2024-12-10T07:15:26,591 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 573523ff7c74144a068617629803d4b6/C of 573523ff7c74144a068617629803d4b6 into 72c7305da8ed424394b9b72b16f77781(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:26,591 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:26,591 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/C, priority=13, startTime=1733814926171; duration=0sec 2024-12-10T07:15:26,591 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:26,591 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:C 2024-12-10T07:15:26,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814986647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:26,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814986658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:26,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814986659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:26,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:26,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:26,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:26,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814986661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:26,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:26,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814986661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:26,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:26,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:26,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814986784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:26,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814986785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:26,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:26,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:26,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814986785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:26,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:26,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814986785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:26,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:26,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:26,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814986786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:26,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:26,856 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121037caa257f8064e2999072db0f715ce31_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121037caa257f8064e2999072db0f715ce31_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:26,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/833ffdadaec44481adf26c4481272a55, store: [table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:26,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/833ffdadaec44481adf26c4481272a55 is 175, key is test_row_0/A:col10/1733814925378/Put/seqid=0 2024-12-10T07:15:26,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741998_1174 (size=31255) 2024-12-10T07:15:26,891 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/833ffdadaec44481adf26c4481272a55 2024-12-10T07:15:26,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/0a065ab0d35d4bcb930075c4dbc396fd is 50, key is test_row_0/B:col10/1733814925378/Put/seqid=0 2024-12-10T07:15:26,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741999_1175 (size=12301) 2024-12-10T07:15:26,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814986995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:26,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:26,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814986995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:26,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:26,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:27,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814986997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:27,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814986997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:27,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814987004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:27,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:27,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:27,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:27,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814987305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:27,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:27,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:27,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:27,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814987306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:27,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:27,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:27,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:27,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814987315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:27,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:27,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:27,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:27,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814987315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:27,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:27,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:27,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:27,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814987318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:27,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:27,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:27,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:27,363 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/0a065ab0d35d4bcb930075c4dbc396fd 2024-12-10T07:15:27,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/4ba6dbfa77d945c389a3fb8f17d8158a is 50, key is test_row_0/C:col10/1733814925378/Put/seqid=0 2024-12-10T07:15:27,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742000_1176 (size=12301) 2024-12-10T07:15:27,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814987816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:27,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814987816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:27,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814987829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:27,845 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/4ba6dbfa77d945c389a3fb8f17d8158a 2024-12-10T07:15:27,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:27,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:27,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:27,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814987838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:27,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814987838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:27,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:27,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:27,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:27,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:27,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:27,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:27,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/833ffdadaec44481adf26c4481272a55 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/833ffdadaec44481adf26c4481272a55 2024-12-10T07:15:27,898 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/833ffdadaec44481adf26c4481272a55, entries=150, sequenceid=324, filesize=30.5 K 2024-12-10T07:15:27,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/0a065ab0d35d4bcb930075c4dbc396fd as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/0a065ab0d35d4bcb930075c4dbc396fd 2024-12-10T07:15:27,921 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/0a065ab0d35d4bcb930075c4dbc396fd, entries=150, sequenceid=324, filesize=12.0 K 2024-12-10T07:15:27,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/4ba6dbfa77d945c389a3fb8f17d8158a as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/4ba6dbfa77d945c389a3fb8f17d8158a 2024-12-10T07:15:27,929 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/4ba6dbfa77d945c389a3fb8f17d8158a, entries=150, sequenceid=324, filesize=12.0 K 2024-12-10T07:15:27,933 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=140.89 KB/144270 for 573523ff7c74144a068617629803d4b6 in 1613ms, sequenceid=324, compaction requested=false 2024-12-10T07:15:27,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:27,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:27,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-10T07:15:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-10T07:15:27,943 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-10T07:15:27,943 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7050 sec 2024-12-10T07:15:27,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 3.7170 sec 2024-12-10T07:15:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-10T07:15:28,397 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:15:28,398 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:15:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-10T07:15:28,401 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:15:28,404 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:15:28,404 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:15:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T07:15:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T07:15:28,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-10T07:15:28,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:28,561 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 573523ff7c74144a068617629803d4b6 3/3 column families, dataSize=140.89 KB heapSize=419.81 KB 2024-12-10T07:15:28,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=A 2024-12-10T07:15:28,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:28,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=B 2024-12-10T07:15:28,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:28,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=C 2024-12-10T07:15:28,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:28,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210b1b61e4165e647e28dc86307030b1d50_573523ff7c74144a068617629803d4b6 is 50, key is test_row_0/A:col10/1733814926637/Put/seqid=0 2024-12-10T07:15:28,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742001_1177 (size=12454) 2024-12-10T07:15:28,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:28,638 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210b1b61e4165e647e28dc86307030b1d50_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b1b61e4165e647e28dc86307030b1d50_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:28,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/fd740df15e0f4620a9e0bd1e11a9b0ba, store: [table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:28,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/fd740df15e0f4620a9e0bd1e11a9b0ba is 175, key is test_row_0/A:col10/1733814926637/Put/seqid=0 2024-12-10T07:15:28,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742002_1178 (size=31255) 2024-12-10T07:15:28,688 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=351, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/fd740df15e0f4620a9e0bd1e11a9b0ba 2024-12-10T07:15:28,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/c8f4a7784869406689dc2ae993d56c14 is 50, key is test_row_0/B:col10/1733814926637/Put/seqid=0 2024-12-10T07:15:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T07:15:28,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742003_1179 (size=12301) 2024-12-10T07:15:28,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:28,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814988869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:28,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814988873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:28,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814988873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:28,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:28,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814988877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:28,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:28,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814988877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:28,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:28,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:28,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:28,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:28,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:28,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:28,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:28,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:28,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:28,881 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_2' in 573523ff7c74144a068617629803d4b6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:15:28,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:28,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:28,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:28,881 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 573523ff7c74144a068617629803d4b6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:15:28,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:28,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:28,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:28,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:28,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814988985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:28,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:28,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814988985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:28,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814988985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:28,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:28,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:28,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T07:15:29,162 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/c8f4a7784869406689dc2ae993d56c14 2024-12-10T07:15:29,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/f4cbafda7d3c4fadb6e0c7db101c4ab0 is 50, key is test_row_0/C:col10/1733814926637/Put/seqid=0 2024-12-10T07:15:29,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814989209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:29,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:29,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:29,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:29,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814989209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:29,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814989213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:29,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:29,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:29,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:29,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:29,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:29,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:29,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742004_1180 (size=12301) 2024-12-10T07:15:29,252 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/f4cbafda7d3c4fadb6e0c7db101c4ab0 2024-12-10T07:15:29,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/fd740df15e0f4620a9e0bd1e11a9b0ba as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/fd740df15e0f4620a9e0bd1e11a9b0ba 2024-12-10T07:15:29,318 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/fd740df15e0f4620a9e0bd1e11a9b0ba, entries=150, sequenceid=351, filesize=30.5 K 2024-12-10T07:15:29,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/c8f4a7784869406689dc2ae993d56c14 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/c8f4a7784869406689dc2ae993d56c14 2024-12-10T07:15:29,337 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/c8f4a7784869406689dc2ae993d56c14, entries=150, sequenceid=351, filesize=12.0 K 2024-12-10T07:15:29,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/f4cbafda7d3c4fadb6e0c7db101c4ab0 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/f4cbafda7d3c4fadb6e0c7db101c4ab0 2024-12-10T07:15:29,359 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/f4cbafda7d3c4fadb6e0c7db101c4ab0, entries=150, sequenceid=351, filesize=12.0 K 2024-12-10T07:15:29,362 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~419.77 KB/429840, currentSize=40.25 KB/41220 for 573523ff7c74144a068617629803d4b6 in 800ms, sequenceid=351, compaction requested=true 2024-12-10T07:15:29,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:29,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:29,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-10T07:15:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-10T07:15:29,373 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-10T07:15:29,373 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 963 msec 2024-12-10T07:15:29,376 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 976 msec 2024-12-10T07:15:29,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 573523ff7c74144a068617629803d4b6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T07:15:29,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=A 2024-12-10T07:15:29,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:29,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=B 2024-12-10T07:15:29,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:29,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=C 2024-12-10T07:15:29,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:29,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-10T07:15:29,548 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:15:29,549 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:15:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-10T07:15:29,551 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:15:29,552 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:15:29,552 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:15:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T07:15:29,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121024b327912e15420580a4ad5f773d2e81_573523ff7c74144a068617629803d4b6 is 50, key is test_row_0/A:col10/1733814929526/Put/seqid=0 2024-12-10T07:15:29,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742005_1181 (size=14994) 2024-12-10T07:15:29,636 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:29,642 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121024b327912e15420580a4ad5f773d2e81_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121024b327912e15420580a4ad5f773d2e81_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:29,645 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/eea4b1090ec048839820ae465872d500, store: [table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:29,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/eea4b1090ec048839820ae465872d500 is 175, key is test_row_0/A:col10/1733814929526/Put/seqid=0 2024-12-10T07:15:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T07:15:29,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742006_1182 (size=39949) 2024-12-10T07:15:29,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733814929629, max timestamp=1733814929672Num uniques 859; 2024-12-10T07:15:29,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-10T07:15:29,680 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=361, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/eea4b1090ec048839820ae465872d500 2024-12-10T07:15:29,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814929629, max timestamp=1733814929672Num uniques 859; 2024-12-10T07:15:29,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:15:29,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814929629, max timestamp=1733814929672Num uniques 859; 2024-12-10T07:15:29,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:15:29,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T07:15:29,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:29,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:29,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:29,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:29,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:29,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/c7e02b0de145415c830acfd9be72bdbb is 50, key is test_row_0/B:col10/1733814929526/Put/seqid=0 2024-12-10T07:15:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:29,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742007_1183 (size=12301) 2024-12-10T07:15:29,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/c7e02b0de145415c830acfd9be72bdbb 2024-12-10T07:15:29,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/48adc0497b934849bf23a7e58ee60046 is 50, key is test_row_0/C:col10/1733814929526/Put/seqid=0 2024-12-10T07:15:29,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742008_1184 (size=12301) 2024-12-10T07:15:29,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/48adc0497b934849bf23a7e58ee60046 2024-12-10T07:15:29,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:29,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814989777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:29,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:29,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:29,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:29,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814989783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:29,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:29,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:29,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814989784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:29,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:29,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:29,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/eea4b1090ec048839820ae465872d500 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/eea4b1090ec048839820ae465872d500 2024-12-10T07:15:29,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/eea4b1090ec048839820ae465872d500, entries=200, sequenceid=361, filesize=39.0 K 2024-12-10T07:15:29,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/c7e02b0de145415c830acfd9be72bdbb as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/c7e02b0de145415c830acfd9be72bdbb 2024-12-10T07:15:29,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T07:15:29,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:29,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:29,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:29,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:29,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:29,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:29,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/c7e02b0de145415c830acfd9be72bdbb, entries=150, sequenceid=361, filesize=12.0 K 2024-12-10T07:15:29,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/48adc0497b934849bf23a7e58ee60046 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/48adc0497b934849bf23a7e58ee60046 2024-12-10T07:15:29,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/48adc0497b934849bf23a7e58ee60046, entries=150, sequenceid=361, filesize=12.0 K 2024-12-10T07:15:29,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T07:15:29,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 573523ff7c74144a068617629803d4b6 in 345ms, sequenceid=361, compaction requested=true 2024-12-10T07:15:29,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:29,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:15:29,875 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:15:29,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:29,876 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:15:29,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:15:29,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:29,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:15:29,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:29,877 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:15:29,877 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:15:29,877 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/A is initiating minor compaction (all files) 2024-12-10T07:15:29,877 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/B is initiating minor compaction (all files) 2024-12-10T07:15:29,877 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/A in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:29,877 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/B in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:29,877 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/08ad69f7ebd14dd1aba06d268acb548c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/833ffdadaec44481adf26c4481272a55, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/fd740df15e0f4620a9e0bd1e11a9b0ba, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/eea4b1090ec048839820ae465872d500] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=131.0 K 2024-12-10T07:15:29,877 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/fd9ed32eb9c04dc098c9bcbc89a3de2f, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/0a065ab0d35d4bcb930075c4dbc396fd, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/c8f4a7784869406689dc2ae993d56c14, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/c7e02b0de145415c830acfd9be72bdbb] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=48.5 K 2024-12-10T07:15:29,878 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:29,878 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. files: [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/08ad69f7ebd14dd1aba06d268acb548c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/833ffdadaec44481adf26c4481272a55, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/fd740df15e0f4620a9e0bd1e11a9b0ba, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/eea4b1090ec048839820ae465872d500] 2024-12-10T07:15:29,878 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08ad69f7ebd14dd1aba06d268acb548c, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733814924156 2024-12-10T07:15:29,878 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting fd9ed32eb9c04dc098c9bcbc89a3de2f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733814924156 2024-12-10T07:15:29,879 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 833ffdadaec44481adf26c4481272a55, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733814925378 2024-12-10T07:15:29,879 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a065ab0d35d4bcb930075c4dbc396fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733814925378 2024-12-10T07:15:29,880 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd740df15e0f4620a9e0bd1e11a9b0ba, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733814926637 2024-12-10T07:15:29,880 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting c8f4a7784869406689dc2ae993d56c14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733814926637 2024-12-10T07:15:29,881 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting eea4b1090ec048839820ae465872d500, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733814928844 2024-12-10T07:15:29,881 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting c7e02b0de145415c830acfd9be72bdbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733814928844 2024-12-10T07:15:29,901 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#B#compaction#152 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:29,902 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/5a507d6eb0d64a7499f5f242515591b8 is 50, key is test_row_0/B:col10/1733814929526/Put/seqid=0 2024-12-10T07:15:29,913 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:29,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 573523ff7c74144a068617629803d4b6 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-12-10T07:15:29,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:29,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=A 2024-12-10T07:15:29,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T07:15:29,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=B 2024-12-10T07:15:29,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T07:15:29,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=C 2024-12-10T07:15:29,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T07:15:29,939 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210c4bfd40f9e334bc9928bff54c4f695ce_573523ff7c74144a068617629803d4b6 store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:29,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742009_1185 (size=12881) 2024-12-10T07:15:29,942 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210c4bfd40f9e334bc9928bff54c4f695ce_573523ff7c74144a068617629803d4b6, store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:29,942 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210c4bfd40f9e334bc9928bff54c4f695ce_573523ff7c74144a068617629803d4b6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:29,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:29,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814989935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:29,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:29,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:29,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:29,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210e1491ba1cb8e4011bde07d94ea323b7d_573523ff7c74144a068617629803d4b6 is 50, key is test_row_0/A:col10/1733814929742/Put/seqid=0 2024-12-10T07:15:29,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:29,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814989949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:29,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:29,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814989953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:29,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:29,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:29,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:29,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:29,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:29,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:29,993 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/5a507d6eb0d64a7499f5f242515591b8 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/5a507d6eb0d64a7499f5f242515591b8 2024-12-10T07:15:30,002 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 573523ff7c74144a068617629803d4b6/B of 573523ff7c74144a068617629803d4b6 into 5a507d6eb0d64a7499f5f242515591b8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:30,002 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:30,002 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/B, priority=12, startTime=1733814929875; duration=0sec 2024-12-10T07:15:30,003 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:30,003 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:B 2024-12-10T07:15:30,003 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:15:30,008 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:15:30,009 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/C is initiating minor compaction (all files) 2024-12-10T07:15:30,009 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/C in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:30,009 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/72c7305da8ed424394b9b72b16f77781, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/4ba6dbfa77d945c389a3fb8f17d8158a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/f4cbafda7d3c4fadb6e0c7db101c4ab0, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/48adc0497b934849bf23a7e58ee60046] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=48.5 K 2024-12-10T07:15:30,009 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 72c7305da8ed424394b9b72b16f77781, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733814924156 2024-12-10T07:15:30,010 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ba6dbfa77d945c389a3fb8f17d8158a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733814925378 2024-12-10T07:15:30,010 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting f4cbafda7d3c4fadb6e0c7db101c4ab0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733814926637 2024-12-10T07:15:30,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742010_1186 (size=4469) 2024-12-10T07:15:30,012 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 48adc0497b934849bf23a7e58ee60046, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733814928844 2024-12-10T07:15:30,015 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#A#compaction#153 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:30,016 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/bfa3b21ede7640479e10f40ae22940ee is 175, key is test_row_0/A:col10/1733814929526/Put/seqid=0 2024-12-10T07:15:30,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T07:15:30,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:30,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:30,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:30,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:30,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:30,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:30,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742011_1187 (size=17534) 2024-12-10T07:15:30,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742012_1188 (size=31835) 2024-12-10T07:15:30,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814990057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:30,073 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#C#compaction#155 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:30,073 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/36ae73aeaccd495fb7e14afee0afc470 is 50, key is test_row_0/C:col10/1733814929526/Put/seqid=0 2024-12-10T07:15:30,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:30,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814990082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:30,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:30,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:30,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:30,095 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/bfa3b21ede7640479e10f40ae22940ee as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/bfa3b21ede7640479e10f40ae22940ee 2024-12-10T07:15:30,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:30,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814990085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:30,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:30,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:30,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:30,103 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 573523ff7c74144a068617629803d4b6/A of 573523ff7c74144a068617629803d4b6 into bfa3b21ede7640479e10f40ae22940ee(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:30,103 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:30,103 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/A, priority=12, startTime=1733814929875; duration=0sec 2024-12-10T07:15:30,105 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:30,105 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:A 2024-12-10T07:15:30,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742013_1189 (size=12881) 2024-12-10T07:15:30,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T07:15:30,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:30,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:30,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:30,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:30,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:30,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T07:15:30,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:30,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814990277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:30,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:30,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:30,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:30,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:30,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814990309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:30,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:30,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814990312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:30,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:30,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:30,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:30,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:30,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:30,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T07:15:30,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:30,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:30,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:30,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:30,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:30,455 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,464 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210e1491ba1cb8e4011bde07d94ea323b7d_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210e1491ba1cb8e4011bde07d94ea323b7d_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:30,466 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/ba3290cd8cf7425c8cc5967c7e1352a9, store: [table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:30,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/ba3290cd8cf7425c8cc5967c7e1352a9 is 175, key is test_row_0/A:col10/1733814929742/Put/seqid=0 2024-12-10T07:15:30,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T07:15:30,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:30,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:30,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:30,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:30,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:30,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:30,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742014_1190 (size=48639) 2024-12-10T07:15:30,522 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=399, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/ba3290cd8cf7425c8cc5967c7e1352a9 2024-12-10T07:15:30,534 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/36ae73aeaccd495fb7e14afee0afc470 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/36ae73aeaccd495fb7e14afee0afc470 2024-12-10T07:15:30,546 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 573523ff7c74144a068617629803d4b6/C of 573523ff7c74144a068617629803d4b6 into 36ae73aeaccd495fb7e14afee0afc470(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:30,546 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:30,547 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/C, priority=12, startTime=1733814929876; duration=0sec 2024-12-10T07:15:30,547 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:30,547 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:C 2024-12-10T07:15:30,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/030fb5dbf2e84cd094e4cb216813fd3b is 50, key is test_row_0/B:col10/1733814929742/Put/seqid=0 2024-12-10T07:15:30,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814990596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:30,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:30,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:30,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:30,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742015_1191 (size=12301) 2024-12-10T07:15:30,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:30,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814990625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:30,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/030fb5dbf2e84cd094e4cb216813fd3b 2024-12-10T07:15:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:30,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:30,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814990641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:30,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/2a03d932a89f4a98b2cd6c995e5143e5 is 50, key is test_row_0/C:col10/1733814929742/Put/seqid=0 2024-12-10T07:15:30,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T07:15:30,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:30,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:30,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:30,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:30,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:30,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T07:15:30,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742016_1192 (size=12301) 2024-12-10T07:15:30,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/2a03d932a89f4a98b2cd6c995e5143e5 2024-12-10T07:15:30,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/ba3290cd8cf7425c8cc5967c7e1352a9 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/ba3290cd8cf7425c8cc5967c7e1352a9 2024-12-10T07:15:30,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/ba3290cd8cf7425c8cc5967c7e1352a9, entries=250, sequenceid=399, filesize=47.5 K 2024-12-10T07:15:30,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/030fb5dbf2e84cd094e4cb216813fd3b as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/030fb5dbf2e84cd094e4cb216813fd3b 2024-12-10T07:15:30,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/030fb5dbf2e84cd094e4cb216813fd3b, entries=150, sequenceid=399, filesize=12.0 K 2024-12-10T07:15:30,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/2a03d932a89f4a98b2cd6c995e5143e5 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/2a03d932a89f4a98b2cd6c995e5143e5 2024-12-10T07:15:30,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/2a03d932a89f4a98b2cd6c995e5143e5, entries=150, sequenceid=399, filesize=12.0 K 2024-12-10T07:15:30,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~436.03 KB/446496, currentSize=33.54 KB/34350 for 573523ff7c74144a068617629803d4b6 in 859ms, sequenceid=399, compaction requested=false 2024-12-10T07:15:30,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:30,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-10T07:15:30,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:30,845 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 573523ff7c74144a068617629803d4b6 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T07:15:30,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=A 2024-12-10T07:15:30,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:30,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=B 2024-12-10T07:15:30,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:30,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=C 2024-12-10T07:15:30,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:30,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121038c2eafbf08548c0b36eb63bbfb124f3_573523ff7c74144a068617629803d4b6 is 50, key is test_row_0/A:col10/1733814929943/Put/seqid=0 2024-12-10T07:15:30,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742017_1193 (size=9914) 2024-12-10T07:15:30,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:30,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,929 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121038c2eafbf08548c0b36eb63bbfb124f3_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121038c2eafbf08548c0b36eb63bbfb124f3_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:30,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/7672041d67fc42c8aaba5481403002b1, store: [table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:30,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/7672041d67fc42c8aaba5481403002b1 is 175, key is test_row_0/A:col10/1733814929943/Put/seqid=0 2024-12-10T07:15:30,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:30,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742018_1194 (size=22561) 2024-12-10T07:15:30,998 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=410, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/7672041d67fc42c8aaba5481403002b1 2024-12-10T07:15:31,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/9db61db519dc4834b4fc42e6ef0f6c89 is 50, key is test_row_0/B:col10/1733814929943/Put/seqid=0 2024-12-10T07:15:31,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742019_1195 (size=9857) 2024-12-10T07:15:31,058 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/9db61db519dc4834b4fc42e6ef0f6c89 2024-12-10T07:15:31,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814930968, max timestamp=1733814931063Num uniques 859; 2024-12-10T07:15:31,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:15:31,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/2e0a6b11cc054fec9619438fc2df139d is 50, key is test_row_0/C:col10/1733814929943/Put/seqid=0 2024-12-10T07:15:31,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742020_1196 (size=9857) 2024-12-10T07:15:31,112 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/2e0a6b11cc054fec9619438fc2df139d 2024-12-10T07:15:31,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814991104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:31,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814991104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:31,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:31,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:31,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:31,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:31,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:31,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:31,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814991118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:31,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:31,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:31,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:31,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/7672041d67fc42c8aaba5481403002b1 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/7672041d67fc42c8aaba5481403002b1 2024-12-10T07:15:31,132 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/7672041d67fc42c8aaba5481403002b1, entries=100, sequenceid=410, filesize=22.0 K 2024-12-10T07:15:31,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/9db61db519dc4834b4fc42e6ef0f6c89 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/9db61db519dc4834b4fc42e6ef0f6c89 2024-12-10T07:15:31,139 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/9db61db519dc4834b4fc42e6ef0f6c89, entries=100, sequenceid=410, filesize=9.6 K 2024-12-10T07:15:31,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/2e0a6b11cc054fec9619438fc2df139d as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/2e0a6b11cc054fec9619438fc2df139d 2024-12-10T07:15:31,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814991145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:31,149 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/2e0a6b11cc054fec9619438fc2df139d, entries=100, sequenceid=410, filesize=9.6 K 2024-12-10T07:15:31,151 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=152.07 KB/155720 for 573523ff7c74144a068617629803d4b6 in 305ms, sequenceid=410, compaction requested=true 2024-12-10T07:15:31,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:31,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:31,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-10T07:15:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-10T07:15:31,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-10T07:15:31,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6020 sec 2024-12-10T07:15:31,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.6080 sec 2024-12-10T07:15:31,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 573523ff7c74144a068617629803d4b6 3/3 column families, dataSize=158.78 KB heapSize=466.41 KB 2024-12-10T07:15:31,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=A 2024-12-10T07:15:31,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:31,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=B 2024-12-10T07:15:31,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:31,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=C 2024-12-10T07:15:31,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:31,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210254a85fb940d43b4a257248a799c3fcb_573523ff7c74144a068617629803d4b6 is 50, key is test_row_0/A:col10/1733814931102/Put/seqid=0 2024-12-10T07:15:31,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742021_1197 (size=14994) 2024-12-10T07:15:31,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:31,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814991206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:31,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814991225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:31,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:31,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:31,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:31,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:31,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814991225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:31,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:31,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:31,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:31,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:31,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814991314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:31,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814991434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:31,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814991434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:31,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:31,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:31,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:31,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814991526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:31,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:31,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:31,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:31,609 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,624 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210254a85fb940d43b4a257248a799c3fcb_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210254a85fb940d43b4a257248a799c3fcb_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:31,626 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/576aede2cf5a439a873ba639bfe421d5, store: [table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:31,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/576aede2cf5a439a873ba639bfe421d5 is 175, key is test_row_0/A:col10/1733814931102/Put/seqid=0 2024-12-10T07:15:31,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742022_1198 (size=39949) 2024-12-10T07:15:31,640 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=441, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/576aede2cf5a439a873ba639bfe421d5 2024-12-10T07:15:31,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/0235d077e67d44aaba2bccbf8d56eea9 is 50, key is test_row_0/B:col10/1733814931102/Put/seqid=0 2024-12-10T07:15:31,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742023_1199 (size=12301) 2024-12-10T07:15:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-10T07:15:31,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/0235d077e67d44aaba2bccbf8d56eea9 2024-12-10T07:15:31,715 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:15:31,716 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:15:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-10T07:15:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T07:15:31,720 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:15:31,720 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:15:31,720 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:15:31,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/47066829368b4c6a8568d3c8e79497c1 is 50, key is test_row_0/C:col10/1733814931102/Put/seqid=0 2024-12-10T07:15:31,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:31,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:31,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814991745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:31,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:31,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:31,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:31,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814991745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:31,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:31,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:31,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:31,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742024_1200 (size=12301) 2024-12-10T07:15:31,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/47066829368b4c6a8568d3c8e79497c1 2024-12-10T07:15:31,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/576aede2cf5a439a873ba639bfe421d5 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/576aede2cf5a439a873ba639bfe421d5 2024-12-10T07:15:31,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/576aede2cf5a439a873ba639bfe421d5, entries=200, sequenceid=441, filesize=39.0 K 2024-12-10T07:15:31,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/0235d077e67d44aaba2bccbf8d56eea9 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/0235d077e67d44aaba2bccbf8d56eea9 2024-12-10T07:15:31,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/0235d077e67d44aaba2bccbf8d56eea9, entries=150, sequenceid=441, filesize=12.0 K 2024-12-10T07:15:31,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/47066829368b4c6a8568d3c8e79497c1 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/47066829368b4c6a8568d3c8e79497c1 2024-12-10T07:15:31,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/47066829368b4c6a8568d3c8e79497c1, entries=150, sequenceid=441, filesize=12.0 K 2024-12-10T07:15:31,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T07:15:31,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~158.78 KB/162590, heapSize ~466.36 KB/477552, currentSize=20.13 KB/20610 for 573523ff7c74144a068617629803d4b6 in 653ms, sequenceid=441, compaction requested=true 2024-12-10T07:15:31,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:31,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:15:31,826 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:15:31,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:31,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:15:31,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:31,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:15:31,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:31,826 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:15:31,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,838 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142984 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:15:31,838 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/A is initiating minor compaction (all files) 2024-12-10T07:15:31,838 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/A in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:31,838 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/bfa3b21ede7640479e10f40ae22940ee, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/ba3290cd8cf7425c8cc5967c7e1352a9, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/7672041d67fc42c8aaba5481403002b1, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/576aede2cf5a439a873ba639bfe421d5] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=139.6 K 2024-12-10T07:15:31,838 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:31,839 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. files: [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/bfa3b21ede7640479e10f40ae22940ee, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/ba3290cd8cf7425c8cc5967c7e1352a9, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/7672041d67fc42c8aaba5481403002b1, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/576aede2cf5a439a873ba639bfe421d5] 2024-12-10T07:15:31,839 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47340 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:15:31,839 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfa3b21ede7640479e10f40ae22940ee, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733814928844 2024-12-10T07:15:31,839 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/B is initiating minor compaction (all files) 2024-12-10T07:15:31,839 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/B in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:31,839 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/5a507d6eb0d64a7499f5f242515591b8, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/030fb5dbf2e84cd094e4cb216813fd3b, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/9db61db519dc4834b4fc42e6ef0f6c89, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/0235d077e67d44aaba2bccbf8d56eea9] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=46.2 K 2024-12-10T07:15:31,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,840 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a507d6eb0d64a7499f5f242515591b8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733814928844 2024-12-10T07:15:31,840 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba3290cd8cf7425c8cc5967c7e1352a9, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733814929742 2024-12-10T07:15:31,840 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 030fb5dbf2e84cd094e4cb216813fd3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733814929742 2024-12-10T07:15:31,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,842 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7672041d67fc42c8aaba5481403002b1, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1733814929943 2024-12-10T07:15:31,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,843 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 576aede2cf5a439a873ba639bfe421d5, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1733814931086 2024-12-10T07:15:31,843 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 9db61db519dc4834b4fc42e6ef0f6c89, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1733814929943 2024-12-10T07:15:31,843 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 0235d077e67d44aaba2bccbf8d56eea9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1733814931086 2024-12-10T07:15:31,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,860 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:31,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,871 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#B#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:31,872 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/7f082eb1c8d04d1ab16e2dbb778993ff is 50, key is test_row_0/B:col10/1733814931102/Put/seqid=0 2024-12-10T07:15:31,873 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210429a588220904605af5908094b26c164_573523ff7c74144a068617629803d4b6 store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:31,875 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210429a588220904605af5908094b26c164_573523ff7c74144a068617629803d4b6, store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:31,875 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210429a588220904605af5908094b26c164_573523ff7c74144a068617629803d4b6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:31,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-10T07:15:31,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:31,885 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 573523ff7c74144a068617629803d4b6 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T07:15:31,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=A 2024-12-10T07:15:31,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:31,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=B 2024-12-10T07:15:31,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:31,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=C 2024-12-10T07:15:31,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:31,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:31,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:31,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742025_1201 (size=13017) 2024-12-10T07:15:31,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f2cc2dac52574a168408300e1ba245ec_573523ff7c74144a068617629803d4b6 is 50, key is test_row_0/A:col10/1733814931879/Put/seqid=0 2024-12-10T07:15:31,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,953 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/7f082eb1c8d04d1ab16e2dbb778993ff as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/7f082eb1c8d04d1ab16e2dbb778993ff 2024-12-10T07:15:31,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742026_1202 (size=4469) 2024-12-10T07:15:31,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,963 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#A#compaction#164 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:31,963 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/c35fd6ec90924993a41ed932e33885b5 is 175, key is test_row_0/A:col10/1733814931102/Put/seqid=0 2024-12-10T07:15:31,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742027_1203 (size=17534) 2024-12-10T07:15:31,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,980 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 573523ff7c74144a068617629803d4b6/B of 573523ff7c74144a068617629803d4b6 into 7f082eb1c8d04d1ab16e2dbb778993ff(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:31,980 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:31,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,980 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/B, priority=12, startTime=1733814931826; duration=0sec 2024-12-10T07:15:31,980 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:31,980 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:B 2024-12-10T07:15:31,980 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:15:31,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,982 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f2cc2dac52574a168408300e1ba245ec_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f2cc2dac52574a168408300e1ba245ec_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:31,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/29b8e59e0094438da460a16cd6f13109, store: [table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:31,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/29b8e59e0094438da460a16cd6f13109 is 175, key is test_row_0/A:col10/1733814931879/Put/seqid=0 2024-12-10T07:15:31,986 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47340 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:15:31,986 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/C is initiating minor compaction (all files) 2024-12-10T07:15:31,986 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/C in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:31,986 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/36ae73aeaccd495fb7e14afee0afc470, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/2a03d932a89f4a98b2cd6c995e5143e5, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/2e0a6b11cc054fec9619438fc2df139d, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/47066829368b4c6a8568d3c8e79497c1] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=46.2 K 2024-12-10T07:15:31,988 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 36ae73aeaccd495fb7e14afee0afc470, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733814928844 2024-12-10T07:15:31,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,989 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a03d932a89f4a98b2cd6c995e5143e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733814929742 2024-12-10T07:15:31,990 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e0a6b11cc054fec9619438fc2df139d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1733814929943 2024-12-10T07:15:31,991 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 47066829368b4c6a8568d3c8e79497c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1733814931086 2024-12-10T07:15:31,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:31,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742028_1204 (size=31971) 2024-12-10T07:15:32,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742029_1205 (size=48635) 2024-12-10T07:15:32,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,014 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=450, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/29b8e59e0094438da460a16cd6f13109 2024-12-10T07:15:32,015 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#C#compaction#167 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:32,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,016 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/a213e3bfbc764592a946818b746fd390 is 50, key is test_row_0/C:col10/1733814931102/Put/seqid=0 2024-12-10T07:15:32,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,022 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/c35fd6ec90924993a41ed932e33885b5 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/c35fd6ec90924993a41ed932e33885b5 2024-12-10T07:15:32,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T07:15:32,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,048 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 573523ff7c74144a068617629803d4b6/A of 573523ff7c74144a068617629803d4b6 into c35fd6ec90924993a41ed932e33885b5(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:32,049 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:32,049 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/A, priority=12, startTime=1733814931826; duration=0sec 2024-12-10T07:15:32,049 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:32,049 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:A 2024-12-10T07:15:32,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/c847e2c54ffe430e82d6001462a0bd9c is 50, key is test_row_0/B:col10/1733814931879/Put/seqid=0 2024-12-10T07:15:32,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742030_1206 (size=13017) 2024-12-10T07:15:32,104 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/a213e3bfbc764592a946818b746fd390 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/a213e3bfbc764592a946818b746fd390 2024-12-10T07:15:32,112 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 573523ff7c74144a068617629803d4b6/C of 573523ff7c74144a068617629803d4b6 into a213e3bfbc764592a946818b746fd390(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:32,112 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:32,112 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/C, priority=12, startTime=1733814931826; duration=0sec 2024-12-10T07:15:32,112 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:32,112 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:C 2024-12-10T07:15:32,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742031_1207 (size=12297) 2024-12-10T07:15:32,139 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/c847e2c54ffe430e82d6001462a0bd9c 2024-12-10T07:15:32,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/d717ecdd57d04ef093a64293a6825478 is 50, key is test_row_0/C:col10/1733814931879/Put/seqid=0 2024-12-10T07:15:32,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814932067, max timestamp=1733814932160Num uniques 859; 2024-12-10T07:15:32,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:15:32,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814932067, max timestamp=1733814932160Num uniques 859; 2024-12-10T07:15:32,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:15:32,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814932067, max timestamp=1733814932160Num uniques 859; 2024-12-10T07:15:32,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:15:32,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742032_1208 (size=9857) 2024-12-10T07:15:32,204 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/d717ecdd57d04ef093a64293a6825478 2024-12-10T07:15:32,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/29b8e59e0094438da460a16cd6f13109 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/29b8e59e0094438da460a16cd6f13109 2024-12-10T07:15:32,236 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/29b8e59e0094438da460a16cd6f13109, entries=250, sequenceid=450, filesize=47.5 K 2024-12-10T07:15:32,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/c847e2c54ffe430e82d6001462a0bd9c as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/c847e2c54ffe430e82d6001462a0bd9c 2024-12-10T07:15:32,249 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/c847e2c54ffe430e82d6001462a0bd9c, entries=150, sequenceid=450, filesize=12.0 K 2024-12-10T07:15:32,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/d717ecdd57d04ef093a64293a6825478 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/d717ecdd57d04ef093a64293a6825478 2024-12-10T07:15:32,260 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/d717ecdd57d04ef093a64293a6825478, entries=100, sequenceid=450, filesize=9.6 K 2024-12-10T07:15:32,261 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=127.47 KB/130530 for 573523ff7c74144a068617629803d4b6 in 376ms, sequenceid=450, compaction requested=false 2024-12-10T07:15:32,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:32,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:32,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:32,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-10T07:15:32,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 573523ff7c74144a068617629803d4b6 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-10T07:15:32,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-10T07:15:32,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=A 2024-12-10T07:15:32,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:32,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=B 2024-12-10T07:15:32,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:32,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=C 2024-12-10T07:15:32,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:32,275 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-10T07:15:32,275 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 548 msec 2024-12-10T07:15:32,280 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 560 msec 2024-12-10T07:15:32,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:32,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:32,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814992275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:32,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814992281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:32,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:32,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814992281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:32,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:32,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814992282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:32,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:32,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:32,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:32,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814992297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:32,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412107447ac47e74c42578ed77393bb65cfcb_573523ff7c74144a068617629803d4b6 is 50, key is test_row_0/A:col10/1733814932253/Put/seqid=0 2024-12-10T07:15:32,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-10T07:15:32,348 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:15:32,349 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:15:32,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742033_1209 (size=14994) 2024-12-10T07:15:32,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-10T07:15:32,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T07:15:32,352 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:32,352 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:15:32,354 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:15:32,354 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:15:32,361 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412107447ac47e74c42578ed77393bb65cfcb_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412107447ac47e74c42578ed77393bb65cfcb_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:32,362 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/1ca04cefafc7437e903a5d8efbc4a78c, store: [table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:32,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/1ca04cefafc7437e903a5d8efbc4a78c is 175, key is test_row_0/A:col10/1733814932253/Put/seqid=0 2024-12-10T07:15:32,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742034_1210 (size=39949) 2024-12-10T07:15:32,390 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=490, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/1ca04cefafc7437e903a5d8efbc4a78c 2024-12-10T07:15:32,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:32,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:32,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814992405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814992405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:32,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814992409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:32,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:32,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:32,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:32,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814992420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:32,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:32,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/87fd7489c19e45ed9d68180e76499b42 is 50, key is test_row_0/B:col10/1733814932253/Put/seqid=0 2024-12-10T07:15:32,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814992404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:32,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:32,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T07:15:32,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742035_1211 (size=12301) 2024-12-10T07:15:32,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/87fd7489c19e45ed9d68180e76499b42 2024-12-10T07:15:32,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/02a182e187094a08b13a9a4684c5dde4 is 50, key is test_row_0/C:col10/1733814932253/Put/seqid=0 2024-12-10T07:15:32,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:32,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:32,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:32,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:32,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:32,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:32,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:32,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742036_1212 (size=12301) 2024-12-10T07:15:32,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814992625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:32,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:32,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814992625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:32,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:32,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:32,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814992649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:32,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:32,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:32,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814992649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:32,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:32,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:32,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814992649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:32,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:32,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:32,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:32,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:32,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:32,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:32,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:32,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:32,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T07:15:32,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:32,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:32,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:32,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:32,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:32,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:32,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:32,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/02a182e187094a08b13a9a4684c5dde4 2024-12-10T07:15:32,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814992935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:32,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814992935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:32,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/1ca04cefafc7437e903a5d8efbc4a78c as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/1ca04cefafc7437e903a5d8efbc4a78c 2024-12-10T07:15:32,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/1ca04cefafc7437e903a5d8efbc4a78c, entries=200, sequenceid=490, filesize=39.0 K 2024-12-10T07:15:32,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/87fd7489c19e45ed9d68180e76499b42 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/87fd7489c19e45ed9d68180e76499b42 2024-12-10T07:15:32,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/87fd7489c19e45ed9d68180e76499b42, entries=150, sequenceid=490, filesize=12.0 K 2024-12-10T07:15:32,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/02a182e187094a08b13a9a4684c5dde4 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/02a182e187094a08b13a9a4684c5dde4 2024-12-10T07:15:32,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:32,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814992955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:32,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:32,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:32,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/02a182e187094a08b13a9a4684c5dde4, entries=150, sequenceid=490, filesize=12.0 K 2024-12-10T07:15:32,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 573523ff7c74144a068617629803d4b6 in 697ms, sequenceid=490, compaction requested=true 2024-12-10T07:15:32,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:32,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:15:32,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:32,962 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:32,962 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:32,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:15:32,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:32,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:15:32,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:32,963 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120555 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:32,963 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/A is initiating minor compaction (all files) 2024-12-10T07:15:32,963 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/A in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:32,963 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37615 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:32,963 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/B is initiating minor compaction (all files) 2024-12-10T07:15:32,963 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/c35fd6ec90924993a41ed932e33885b5, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/29b8e59e0094438da460a16cd6f13109, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/1ca04cefafc7437e903a5d8efbc4a78c] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=117.7 K 2024-12-10T07:15:32,963 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/B in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:32,963 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:32,963 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. files: [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/c35fd6ec90924993a41ed932e33885b5, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/29b8e59e0094438da460a16cd6f13109, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/1ca04cefafc7437e903a5d8efbc4a78c] 2024-12-10T07:15:32,963 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/7f082eb1c8d04d1ab16e2dbb778993ff, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/c847e2c54ffe430e82d6001462a0bd9c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/87fd7489c19e45ed9d68180e76499b42] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=36.7 K 2024-12-10T07:15:32,964 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting c35fd6ec90924993a41ed932e33885b5, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1733814931086 2024-12-10T07:15:32,964 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f082eb1c8d04d1ab16e2dbb778993ff, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1733814931086 2024-12-10T07:15:32,964 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting c847e2c54ffe430e82d6001462a0bd9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1733814931854 2024-12-10T07:15:32,964 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29b8e59e0094438da460a16cd6f13109, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1733814931186 2024-12-10T07:15:32,965 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ca04cefafc7437e903a5d8efbc4a78c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1733814932243 2024-12-10T07:15:32,965 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 87fd7489c19e45ed9d68180e76499b42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1733814932243 2024-12-10T07:15:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:32,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 573523ff7c74144a068617629803d4b6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T07:15:32,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=A 2024-12-10T07:15:32,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:32,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=B 2024-12-10T07:15:32,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:32,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=C 2024-12-10T07:15:32,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:32,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:32,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:32,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:32,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:32,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:32,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:32,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T07:15:32,982 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#B#compaction#173 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:32,983 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/bc5eff2ac15b4750ad39c19a6be65f43 is 50, key is test_row_0/B:col10/1733814932253/Put/seqid=0 2024-12-10T07:15:32,984 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:33,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121053d0d9562efb4b0e8faf5e333da44cc8_573523ff7c74144a068617629803d4b6 is 50, key is test_row_0/A:col10/1733814932278/Put/seqid=0 2024-12-10T07:15:33,005 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210e8b7809436ce48248199c594f773c92a_573523ff7c74144a068617629803d4b6 store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:33,007 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210e8b7809436ce48248199c594f773c92a_573523ff7c74144a068617629803d4b6, store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:33,007 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210e8b7809436ce48248199c594f773c92a_573523ff7c74144a068617629803d4b6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:33,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742037_1213 (size=13119) 2024-12-10T07:15:33,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742038_1214 (size=12454) 2024-12-10T07:15:33,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733814932999, max timestamp=1733814933027Num uniques 859; 2024-12-10T07:15:33,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T07:15:33,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733814932999, max timestamp=1733814933027Num uniques 859; 2024-12-10T07:15:33,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T07:15:33,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733814932999, max timestamp=1733814933027Num uniques 859; 2024-12-10T07:15:33,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T07:15:33,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742039_1215 (size=4469) 2024-12-10T07:15:33,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:33,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814993063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:33,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:33,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814993067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:33,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:33,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:33,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:33,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:33,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:33,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:33,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:33,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:33,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814993175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:33,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:33,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:33,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:33,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:33,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814993185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:33,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:33,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:33,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:33,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:33,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:33,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:33,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:33,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:33,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:33,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:33,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:33,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814993385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:33,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:33,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:33,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:33,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:33,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814993394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:33,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:33,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:33,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:33,427 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:33,433 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121053d0d9562efb4b0e8faf5e333da44cc8_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121053d0d9562efb4b0e8faf5e333da44cc8_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:33,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:33,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:33,434 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/3f956f4adfa047a29f991c4267e9b913, store: [table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:33,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:33,434 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/bc5eff2ac15b4750ad39c19a6be65f43 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/bc5eff2ac15b4750ad39c19a6be65f43 2024-12-10T07:15:33,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:33,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:33,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:33,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/3f956f4adfa047a29f991c4267e9b913 is 175, key is test_row_0/A:col10/1733814932278/Put/seqid=0 2024-12-10T07:15:33,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:33,440 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#A#compaction#174 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:33,441 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/04b8ad20c2e94ed1be12e579b576d745 is 175, key is test_row_0/A:col10/1733814932253/Put/seqid=0 2024-12-10T07:15:33,446 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 573523ff7c74144a068617629803d4b6/B of 573523ff7c74144a068617629803d4b6 into bc5eff2ac15b4750ad39c19a6be65f43(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:33,446 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:33,446 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/B, priority=13, startTime=1733814932962; duration=0sec 2024-12-10T07:15:33,446 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:33,446 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:B 2024-12-10T07:15:33,446 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:33,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:33,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:33,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814993445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:33,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814993445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:33,452 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:33,452 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/C is initiating minor compaction (all files) 2024-12-10T07:15:33,452 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/C in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:33,452 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/a213e3bfbc764592a946818b746fd390, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/d717ecdd57d04ef093a64293a6825478, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/02a182e187094a08b13a9a4684c5dde4] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=34.4 K 2024-12-10T07:15:33,453 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting a213e3bfbc764592a946818b746fd390, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1733814931086 2024-12-10T07:15:33,453 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting d717ecdd57d04ef093a64293a6825478, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1733814931854 2024-12-10T07:15:33,455 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 02a182e187094a08b13a9a4684c5dde4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1733814932243 2024-12-10T07:15:33,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742040_1216 (size=31255) 2024-12-10T07:15:33,458 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=501, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/3f956f4adfa047a29f991c4267e9b913 2024-12-10T07:15:33,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:33,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814993465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:33,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:33,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:33,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:33,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T07:15:33,487 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#C#compaction#176 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:33,487 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/600fef4dc9f54bb8854ec133a95720f4 is 50, key is test_row_0/C:col10/1733814932253/Put/seqid=0 2024-12-10T07:15:33,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/a16d5ea619f342af9c3e5e5d5fd1a2a6 is 50, key is test_row_0/B:col10/1733814932278/Put/seqid=0 2024-12-10T07:15:33,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742041_1217 (size=32073) 2024-12-10T07:15:33,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742042_1218 (size=13119) 2024-12-10T07:15:33,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742043_1219 (size=12301) 2024-12-10T07:15:33,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/a16d5ea619f342af9c3e5e5d5fd1a2a6 2024-12-10T07:15:33,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/2f6487d9109c4b8e9a5e877c305c4e0a is 50, key is test_row_0/C:col10/1733814932278/Put/seqid=0 2024-12-10T07:15:33,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742044_1220 (size=12301) 2024-12-10T07:15:33,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:33,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:33,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:33,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/2f6487d9109c4b8e9a5e877c305c4e0a 2024-12-10T07:15:33,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:33,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:33,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:33,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:33,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/3f956f4adfa047a29f991c4267e9b913 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/3f956f4adfa047a29f991c4267e9b913 2024-12-10T07:15:33,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/3f956f4adfa047a29f991c4267e9b913, entries=150, sequenceid=501, filesize=30.5 K 2024-12-10T07:15:33,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/a16d5ea619f342af9c3e5e5d5fd1a2a6 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/a16d5ea619f342af9c3e5e5d5fd1a2a6 2024-12-10T07:15:33,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/a16d5ea619f342af9c3e5e5d5fd1a2a6, entries=150, sequenceid=501, filesize=12.0 K 2024-12-10T07:15:33,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/2f6487d9109c4b8e9a5e877c305c4e0a as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/2f6487d9109c4b8e9a5e877c305c4e0a 2024-12-10T07:15:33,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/2f6487d9109c4b8e9a5e877c305c4e0a, entries=150, sequenceid=501, filesize=12.0 K 2024-12-10T07:15:33,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 573523ff7c74144a068617629803d4b6 in 666ms, sequenceid=501, compaction requested=false 2024-12-10T07:15:33,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:33,710 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T07:15:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:33,718 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 573523ff7c74144a068617629803d4b6 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-10T07:15:33,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733814933058, max timestamp=1733814933715Num uniques 1018; 2024-12-10T07:15:33,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733814933058, max timestamp=1733814933715Num uniques 1018; 2024-12-10T07:15:33,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-10T07:15:33,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-10T07:15:33,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733814933058, max timestamp=1733814933715Num uniques 1018; 2024-12-10T07:15:33,722 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-10T07:15:33,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=A 2024-12-10T07:15:33,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:33,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=B 2024-12-10T07:15:33,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:33,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=C 2024-12-10T07:15:33,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:33,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a78dfea7e5a94436952763fff64ad364_573523ff7c74144a068617629803d4b6 is 50, key is test_row_0/A:col10/1733814933714/Put/seqid=0 2024-12-10T07:15:33,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742045_1221 (size=17534) 2024-12-10T07:15:33,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:33,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:33,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:33,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:33,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:33,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:33,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:33,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814933758, max timestamp=1733814933772Num uniques 859; 2024-12-10T07:15:33,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:15:33,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814933758, max timestamp=1733814933772Num uniques 859; 2024-12-10T07:15:33,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:15:33,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814933758, max timestamp=1733814933772Num uniques 859; 2024-12-10T07:15:33,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:15:33,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733814933808, max timestamp=1733814933833Num uniques 1068; 2024-12-10T07:15:33,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733814933808, max timestamp=1733814933833Num uniques 1068; 2024-12-10T07:15:33,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312  suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-12-10T07:15:33,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312  suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-12-10T07:15:33,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733814933808, max timestamp=1733814933833Num uniques 1068; 2024-12-10T07:15:33,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312  suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-12-10T07:15:33,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733814933876, max timestamp=1733814933894Num uniques 1177; 2024-12-10T07:15:33,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=10392, suffix heap size=150376,new segment heap size=23848  suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=227 2024-12-10T07:15:33,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:33,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733814933876, max timestamp=1733814933894Num uniques 1177; 2024-12-10T07:15:33,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733814933876, max timestamp=1733814933894Num uniques 1177; 2024-12-10T07:15:33,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:33,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=10392, suffix heap size=150376,new segment heap size=23848  suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=227 2024-12-10T07:15:33,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:33,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:33,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:33,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:33,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=10392, suffix heap size=150376,new segment heap size=23848  suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=227 2024-12-10T07:15:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:33,928 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/04b8ad20c2e94ed1be12e579b576d745 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/04b8ad20c2e94ed1be12e579b576d745 2024-12-10T07:15:33,936 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 573523ff7c74144a068617629803d4b6/A of 573523ff7c74144a068617629803d4b6 into 04b8ad20c2e94ed1be12e579b576d745(size=31.3 K), total size for store is 61.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:33,936 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:33,936 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/A, priority=13, startTime=1733814932961; duration=0sec 2024-12-10T07:15:33,936 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:33,936 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:A 2024-12-10T07:15:33,939 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/600fef4dc9f54bb8854ec133a95720f4 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/600fef4dc9f54bb8854ec133a95720f4 2024-12-10T07:15:33,945 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 573523ff7c74144a068617629803d4b6/C of 573523ff7c74144a068617629803d4b6 into 600fef4dc9f54bb8854ec133a95720f4(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:33,945 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:33,945 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/C, priority=13, startTime=1733814932962; duration=0sec 2024-12-10T07:15:33,945 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:33,945 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:C 2024-12-10T07:15:33,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733814933924, max timestamp=1733814933963Num uniques 1086; 2024-12-10T07:15:33,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984  suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-12-10T07:15:33,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733814933924, max timestamp=1733814933963Num uniques 1086; 2024-12-10T07:15:33,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984  suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-12-10T07:15:33,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733814933924, max timestamp=1733814933963Num uniques 1086; 2024-12-10T07:15:33,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984  suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-12-10T07:15:34,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733814934001, max timestamp=1733814934015Num uniques 1145; 2024-12-10T07:15:34,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733814934001, max timestamp=1733814934015Num uniques 1145; 2024-12-10T07:15:34,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520  suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-12-10T07:15:34,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520  suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-12-10T07:15:34,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733814934001, max timestamp=1733814934015Num uniques 1145; 2024-12-10T07:15:34,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520  suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-12-10T07:15:34,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:34,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:34,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:34,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:34,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:34,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:34,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:34,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733814934049, max timestamp=1733814934068Num uniques 1054; 2024-12-10T07:15:34,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456  suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-12-10T07:15:34,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733814934049, max timestamp=1733814934068Num uniques 1054; 2024-12-10T07:15:34,075 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456  suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-12-10T07:15:34,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733814934049, max timestamp=1733814934068Num uniques 1054; 2024-12-10T07:15:34,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456  suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-12-10T07:15:34,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733814934109, max timestamp=1733814934122Num uniques 1063; 2024-12-10T07:15:34,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733814934109, max timestamp=1733814934122Num uniques 1063; 2024-12-10T07:15:34,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192  suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-12-10T07:15:34,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192  suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-12-10T07:15:34,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733814934109, max timestamp=1733814934122Num uniques 1063; 2024-12-10T07:15:34,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192  suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-12-10T07:15:34,137 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:34,145 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a78dfea7e5a94436952763fff64ad364_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a78dfea7e5a94436952763fff64ad364_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:34,146 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/3a36e44d8a4f45e8a1c37881236343db, store: [table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:34,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/3a36e44d8a4f45e8a1c37881236343db is 175, key is test_row_0/A:col10/1733814933714/Put/seqid=0 2024-12-10T07:15:34,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742046_1222 (size=48639) 2024-12-10T07:15:34,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=322, cellSize=14742, totalHeapSize=33728, min timestamp=1733814934135, max timestamp=1733814934171Num uniques 1022; 2024-12-10T07:15:34,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=322, cellSize=14742, totalHeapSize=33728, min timestamp=1733814934135, max timestamp=1733814934171Num uniques 1022; 2024-12-10T07:15:34,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=14742, suffix heap size=134256,new segment heap size=33728  suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=322 2024-12-10T07:15:34,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=14742, suffix heap size=134256,new segment heap size=33728  suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=322 2024-12-10T07:15:34,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=322, cellSize=14742, totalHeapSize=33728, min timestamp=1733814934135, max timestamp=1733814934171Num uniques 1022; 2024-12-10T07:15:34,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=14742, suffix heap size=134256,new segment heap size=33728  suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=322 2024-12-10T07:15:34,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:34,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:34,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:34,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:34,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:34,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:34,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:34,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733814934201, max timestamp=1733814934218Num uniques 1181; 2024-12-10T07:15:34,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=54086, new segment data size=10576, suffix heap size=150792,new segment heap size=24264  suffix off heap size=0, new segment off heap size=0, suffix cells count=1181, new segment cells count=231 2024-12-10T07:15:34,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733814934201, max timestamp=1733814934218Num uniques 1181; 2024-12-10T07:15:34,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=54086, new segment data size=10576, suffix heap size=150792,new segment heap size=24264  suffix off heap size=0, new segment off heap size=0, suffix cells count=1181, new segment cells count=231 2024-12-10T07:15:34,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733814934201, max timestamp=1733814934218Num uniques 1181; 2024-12-10T07:15:34,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=54086, new segment data size=10576, suffix heap size=150792,new segment heap size=24264  suffix off heap size=0, new segment off heap size=0, suffix cells count=1181, new segment cells count=231 2024-12-10T07:15:34,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733814934244, max timestamp=1733814934262Num uniques 1090; 2024-12-10T07:15:34,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200  suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-12-10T07:15:34,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733814934244, max timestamp=1733814934262Num uniques 1090; 2024-12-10T07:15:34,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200  suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-12-10T07:15:34,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733814934244, max timestamp=1733814934262Num uniques 1090; 2024-12-10T07:15:34,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200  suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-12-10T07:15:34,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9104, totalHeapSize=20936, min timestamp=1733814934300, max timestamp=1733814934317Num uniques 1099; 2024-12-10T07:15:34,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50324, new segment data size=9104, suffix heap size=142264,new segment heap size=20936  suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-12-10T07:15:34,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9104, totalHeapSize=20936, min timestamp=1733814934300, max timestamp=1733814934317Num uniques 1099; 2024-12-10T07:15:34,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50324, new segment data size=9104, suffix heap size=142264,new segment heap size=20936  suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-12-10T07:15:34,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9104, totalHeapSize=20936, min timestamp=1733814934300, max timestamp=1733814934317Num uniques 1099; 2024-12-10T07:15:34,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50324, new segment data size=9104, suffix heap size=142264,new segment heap size=20936  suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-12-10T07:15:34,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:34,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:34,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:34,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:34,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:34,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:34,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:34,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1733814934355, max timestamp=1733814934422Num uniques 1058; 2024-12-10T07:15:34,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=9520, suffix heap size=138000,new segment heap size=21872  suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=208 2024-12-10T07:15:34,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1733814934355, max timestamp=1733814934422Num uniques 1058; 2024-12-10T07:15:34,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=9520, suffix heap size=138000,new segment heap size=21872  suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=208 2024-12-10T07:15:34,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1733814934355, max timestamp=1733814934422Num uniques 1058; 2024-12-10T07:15:34,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=9520, suffix heap size=138000,new segment heap size=21872  suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=208 2024-12-10T07:15:34,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12215, totalHeapSize=28008, min timestamp=1733814934448, max timestamp=1733814934472Num uniques 1067; 2024-12-10T07:15:34,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48855, new segment data size=12215, suffix heap size=138936,new segment heap size=28008  suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=267 2024-12-10T07:15:34,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12222, totalHeapSize=28008, min timestamp=1733814934448, max timestamp=1733814934472Num uniques 1067; 2024-12-10T07:15:34,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=12222, suffix heap size=138936,new segment heap size=28008  suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=267 2024-12-10T07:15:34,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12219, totalHeapSize=28008, min timestamp=1733814934448, max timestamp=1733814934472Num uniques 1067; 2024-12-10T07:15:34,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48859, new segment data size=12219, suffix heap size=138936,new segment heap size=28008  suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=267 2024-12-10T07:15:34,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T07:15:34,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=276, cellSize=12636, totalHeapSize=28944, min timestamp=1733814934496, max timestamp=1733814934503Num uniques 1126; 2024-12-10T07:15:34,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51566, new segment data size=12636, suffix heap size=145072,new segment heap size=28944  suffix off heap size=0, new segment off heap size=0, suffix cells count=1126, new segment cells count=276 2024-12-10T07:15:34,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=276, cellSize=12636, totalHeapSize=28944, min timestamp=1733814934496, max timestamp=1733814934503Num uniques 1126; 2024-12-10T07:15:34,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:34,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51566, new segment data size=12636, suffix heap size=145072,new segment heap size=28944  suffix off heap size=0, new segment off heap size=0, suffix cells count=1126, new segment cells count=276 2024-12-10T07:15:34,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814994501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:34,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:34,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=276, cellSize=12636, totalHeapSize=28944, min timestamp=1733814934496, max timestamp=1733814934503Num uniques 1126; 2024-12-10T07:15:34,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51566, new segment data size=12636, suffix heap size=145072,new segment heap size=28944  suffix off heap size=0, new segment off heap size=0, suffix cells count=1126, new segment cells count=276 2024-12-10T07:15:34,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:34,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:34,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:34,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:34,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:34,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:34,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:34,551 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=540, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/3a36e44d8a4f45e8a1c37881236343db 2024-12-10T07:15:34,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/8175457833284fe8a2a1ea20e006c8ec is 50, key is test_row_0/B:col10/1733814933714/Put/seqid=0 2024-12-10T07:15:34,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742047_1223 (size=12301) 2024-12-10T07:15:34,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/8175457833284fe8a2a1ea20e006c8ec 2024-12-10T07:15:34,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:34,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814994563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:34,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:34,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814994564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:34,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1733814934557, max timestamp=1733814934572Num uniques 1135; 2024-12-10T07:15:34,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51980, new segment data size=8470, suffix heap size=146008,new segment heap size=19480  suffix off heap size=0, new segment off heap size=0, suffix cells count=1135, new segment cells count=185 2024-12-10T07:15:34,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733814934557, max timestamp=1733814934572Num uniques 1135; 2024-12-10T07:15:34,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51980, new segment data size=6870, suffix heap size=146008,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=1135, new segment cells count=150 2024-12-10T07:15:34,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1733814934557, max timestamp=1733814934572Num uniques 1135; 2024-12-10T07:15:34,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51980, new segment data size=8470, suffix heap size=146008,new segment heap size=19480  suffix off heap size=0, new segment off heap size=0, suffix cells count=1135, new segment cells count=185 2024-12-10T07:15:34,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/d51f19e13e6b42d18b8e51e9020e4f88 is 50, key is test_row_0/C:col10/1733814933714/Put/seqid=0 2024-12-10T07:15:34,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742048_1224 (size=12301) 2024-12-10T07:15:34,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/d51f19e13e6b42d18b8e51e9020e4f88 2024-12-10T07:15:34,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/3a36e44d8a4f45e8a1c37881236343db as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/3a36e44d8a4f45e8a1c37881236343db 2024-12-10T07:15:34,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/3a36e44d8a4f45e8a1c37881236343db, entries=250, sequenceid=540, filesize=47.5 K 2024-12-10T07:15:34,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/8175457833284fe8a2a1ea20e006c8ec as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/8175457833284fe8a2a1ea20e006c8ec 2024-12-10T07:15:34,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/8175457833284fe8a2a1ea20e006c8ec, entries=150, sequenceid=540, filesize=12.0 K 2024-12-10T07:15:34,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/d51f19e13e6b42d18b8e51e9020e4f88 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/d51f19e13e6b42d18b8e51e9020e4f88 2024-12-10T07:15:34,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/d51f19e13e6b42d18b8e51e9020e4f88, entries=150, sequenceid=540, filesize=12.0 K 2024-12-10T07:15:34,625 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~80.58 KB/82512, currentSize=105.78 KB/108320 for 573523ff7c74144a068617629803d4b6 in 907ms, sequenceid=540, compaction requested=true 2024-12-10T07:15:34,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:34,626 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:34,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:15:34,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:34,626 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:34,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:15:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:34,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:34,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:15:34,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:34,628 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111967 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:34,628 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/A is initiating minor compaction (all files) 2024-12-10T07:15:34,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 573523ff7c74144a068617629803d4b6 3/3 column families, dataSize=119.20 KB heapSize=339.15 KB 2024-12-10T07:15:34,628 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/A in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:34,628 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/04b8ad20c2e94ed1be12e579b576d745, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/3f956f4adfa047a29f991c4267e9b913, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/3a36e44d8a4f45e8a1c37881236343db] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=109.3 K 2024-12-10T07:15:34,628 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:34,628 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. files: [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/04b8ad20c2e94ed1be12e579b576d745, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/3f956f4adfa047a29f991c4267e9b913, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/3a36e44d8a4f45e8a1c37881236343db] 2024-12-10T07:15:34,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=A 2024-12-10T07:15:34,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:34,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=B 2024-12-10T07:15:34,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:34,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=C 2024-12-10T07:15:34,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:34,630 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04b8ad20c2e94ed1be12e579b576d745, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1733814932243 2024-12-10T07:15:34,630 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:34,630 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/B is initiating minor compaction (all files) 2024-12-10T07:15:34,630 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f956f4adfa047a29f991c4267e9b913, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1733814932278 2024-12-10T07:15:34,630 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/B in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:34,630 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/bc5eff2ac15b4750ad39c19a6be65f43, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/a16d5ea619f342af9c3e5e5d5fd1a2a6, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/8175457833284fe8a2a1ea20e006c8ec] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=36.8 K 2024-12-10T07:15:34,631 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a36e44d8a4f45e8a1c37881236343db, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1733814933058 2024-12-10T07:15:34,631 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting bc5eff2ac15b4750ad39c19a6be65f43, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1733814932243 2024-12-10T07:15:34,631 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting a16d5ea619f342af9c3e5e5d5fd1a2a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1733814932278 2024-12-10T07:15:34,632 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 8175457833284fe8a2a1ea20e006c8ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1733814933062 2024-12-10T07:15:34,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108e06f8bca1e44716b72e4538ba296ac6_573523ff7c74144a068617629803d4b6 is 50, key is test_row_0/A:col10/1733814934612/Put/seqid=0 2024-12-10T07:15:34,645 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:34,651 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#B#compaction#184 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:34,651 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/e54c4fda6e8e40518e6a03966208ad5b is 50, key is test_row_0/B:col10/1733814933714/Put/seqid=0 2024-12-10T07:15:34,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814994658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:34,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:34,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:34,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:34,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:34,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:34,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:34,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:34,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814994660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:34,670 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210def7a504830043ce9065d0e86a6ccfae_573523ff7c74144a068617629803d4b6 store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:34,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:34,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814994660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:34,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:34,672 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210def7a504830043ce9065d0e86a6ccfae_573523ff7c74144a068617629803d4b6, store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:34,673 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210def7a504830043ce9065d0e86a6ccfae_573523ff7c74144a068617629803d4b6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:34,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742049_1225 (size=17534) 2024-12-10T07:15:34,677 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:34,685 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108e06f8bca1e44716b72e4538ba296ac6_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108e06f8bca1e44716b72e4538ba296ac6_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:34,689 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/a430374a38df44c5b8f7104361ff88e9, store: [table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:34,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/a430374a38df44c5b8f7104361ff88e9 is 175, key is test_row_0/A:col10/1733814934612/Put/seqid=0 2024-12-10T07:15:34,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814994685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:34,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:34,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814994685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:34,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:34,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742050_1226 (size=13221) 2024-12-10T07:15:34,710 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/e54c4fda6e8e40518e6a03966208ad5b as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/e54c4fda6e8e40518e6a03966208ad5b 2024-12-10T07:15:34,718 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 573523ff7c74144a068617629803d4b6/B of 573523ff7c74144a068617629803d4b6 into e54c4fda6e8e40518e6a03966208ad5b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:34,718 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:34,718 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/B, priority=13, startTime=1733814934626; duration=0sec 2024-12-10T07:15:34,719 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:34,719 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:B 2024-12-10T07:15:34,719 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:34,721 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:34,721 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/C is initiating minor compaction (all files) 2024-12-10T07:15:34,721 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/C in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:34,721 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/600fef4dc9f54bb8854ec133a95720f4, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/2f6487d9109c4b8e9a5e877c305c4e0a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/d51f19e13e6b42d18b8e51e9020e4f88] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=36.8 K 2024-12-10T07:15:34,722 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 600fef4dc9f54bb8854ec133a95720f4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1733814932243 2024-12-10T07:15:34,722 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f6487d9109c4b8e9a5e877c305c4e0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1733814932278 2024-12-10T07:15:34,722 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting d51f19e13e6b42d18b8e51e9020e4f88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1733814933062 2024-12-10T07:15:34,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742051_1227 (size=4469) 2024-12-10T07:15:34,735 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#A#compaction#183 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:34,735 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/cf9a7f44258847cb9124daee45823573 is 175, key is test_row_0/A:col10/1733814933714/Put/seqid=0 2024-12-10T07:15:34,750 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#C#compaction#185 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:34,751 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/c859d5407dcb44aab6d6e782c67cff30 is 50, key is test_row_0/C:col10/1733814933714/Put/seqid=0 2024-12-10T07:15:34,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742052_1228 (size=48639) 2024-12-10T07:15:34,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814994765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:34,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:34,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742054_1230 (size=13221) 2024-12-10T07:15:34,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742053_1229 (size=32175) 2024-12-10T07:15:34,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814994774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:34,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:34,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814994774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:34,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:34,790 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/cf9a7f44258847cb9124daee45823573 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/cf9a7f44258847cb9124daee45823573 2024-12-10T07:15:34,798 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/c859d5407dcb44aab6d6e782c67cff30 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/c859d5407dcb44aab6d6e782c67cff30 2024-12-10T07:15:34,798 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 573523ff7c74144a068617629803d4b6/A of 573523ff7c74144a068617629803d4b6 into cf9a7f44258847cb9124daee45823573(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:34,798 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:34,798 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/A, priority=13, startTime=1733814934625; duration=0sec 2024-12-10T07:15:34,798 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:34,798 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:A 2024-12-10T07:15:34,806 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 573523ff7c74144a068617629803d4b6/C of 573523ff7c74144a068617629803d4b6 into c859d5407dcb44aab6d6e782c67cff30(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:34,806 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:34,806 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/C, priority=13, startTime=1733814934626; duration=0sec 2024-12-10T07:15:34,807 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:34,807 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:C 2024-12-10T07:15:34,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:34,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:34,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:34,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:34,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:34,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:34,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814994895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:34,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:34,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814994895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:34,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:34,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:34,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:34,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:34,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:34,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:34,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:34,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:34,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814994974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:34,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:34,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814994985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:34,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:34,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814994995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:34,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:34,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:35,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:35,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:35,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:35,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:35,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:35,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:35,155 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=818, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/a430374a38df44c5b8f7104361ff88e9 2024-12-10T07:15:35,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/2dcaa46d3dc54ed7bf9d7d4f126e434d is 50, key is test_row_0/B:col10/1733814934612/Put/seqid=0 2024-12-10T07:15:35,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742055_1231 (size=12301) 2024-12-10T07:15:35,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=818 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/2dcaa46d3dc54ed7bf9d7d4f126e434d 2024-12-10T07:15:35,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/a97e82da1f14476cba2937838740b1e9 is 50, key is test_row_0/C:col10/1733814934612/Put/seqid=0 2024-12-10T07:15:35,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742056_1232 (size=12301) 2024-12-10T07:15:35,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814995204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:35,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:35,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:35,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:35,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814995204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:35,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:35,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:35,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:35,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:35,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:35,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:35,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:35,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:35,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:35,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:35,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814995285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:35,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:35,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:35,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:35,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:35,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814995294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:35,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:35,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814995304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:35,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:35,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:35,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:35,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:35,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:35,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:35,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:35,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:35,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:35,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:35,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:35,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:35,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:35,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:35,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:35,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:35,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.93 KB at sequenceid=818 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/a97e82da1f14476cba2937838740b1e9 2024-12-10T07:15:35,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/a430374a38df44c5b8f7104361ff88e9 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/a430374a38df44c5b8f7104361ff88e9 2024-12-10T07:15:35,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/a430374a38df44c5b8f7104361ff88e9, entries=250, sequenceid=818, filesize=47.5 K 2024-12-10T07:15:35,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/2dcaa46d3dc54ed7bf9d7d4f126e434d as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/2dcaa46d3dc54ed7bf9d7d4f126e434d 2024-12-10T07:15:35,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/2dcaa46d3dc54ed7bf9d7d4f126e434d, entries=150, sequenceid=818, filesize=12.0 K 2024-12-10T07:15:35,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/a97e82da1f14476cba2937838740b1e9 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/a97e82da1f14476cba2937838740b1e9 2024-12-10T07:15:35,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/a97e82da1f14476cba2937838740b1e9, entries=150, sequenceid=818, filesize=12.0 K 2024-12-10T07:15:35,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~125.91 KB/128930, heapSize ~359.02 KB/367640, currentSize=53.67 KB/54960 for 573523ff7c74144a068617629803d4b6 in 985ms, sequenceid=818, compaction requested=false 2024-12-10T07:15:35,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:35,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 573523ff7c74144a068617629803d4b6 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T07:15:35,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=A 2024-12-10T07:15:35,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:35,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=B 2024-12-10T07:15:35,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:35,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=C 2024-12-10T07:15:35,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:35,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412105aa2dfc9a70a42feb2e4b925200141c2_573523ff7c74144a068617629803d4b6 is 50, key is test_row_0/A:col10/1733814935715/Put/seqid=0 2024-12-10T07:15:35,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:35,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:35,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:35,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:35,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:35,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:35,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742057_1233 (size=14994) 2024-12-10T07:15:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:35,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:35,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814995743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:35,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:35,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:35,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:35,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814995745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:35,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:35,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:35,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:35,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814995794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:35,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:35,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:35,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:35,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814995805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:35,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:35,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:35,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:35,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814995816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:35,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:35,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:35,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:35,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814995855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:35,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:35,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:35,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:35,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814995855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:35,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:35,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:35,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:35,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:35,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:35,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:35,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:35,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:35,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:35,988 INFO [Thread-614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-614. 2024-12-10T07:15:35,988 DEBUG [Thread-614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:15:35,988 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:35,988 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:35,988 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:15:35,988 INFO [Thread-612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-612. 2024-12-10T07:15:35,988 DEBUG [Thread-612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:15:35,988 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:35,989 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:35,989 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:15:35,989 INFO [Thread-610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-610. 2024-12-10T07:15:35,990 DEBUG [Thread-610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:15:35,990 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:35,990 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:35,990 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:15:35,990 INFO [Thread-608 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-608. 2024-12-10T07:15:35,990 DEBUG [Thread-608 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:15:35,990 DEBUG [Thread-608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:35,990 DEBUG [Thread-608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:35,990 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:15:36,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:36,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:36,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:36,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:36,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:36,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:36,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:36,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:36,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814996065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814996065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:36,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:36,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:36,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:36,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:36,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:36,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:36,144 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:36,150 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412105aa2dfc9a70a42feb2e4b925200141c2_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105aa2dfc9a70a42feb2e4b925200141c2_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:36,152 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/d326f5195a0b44b0928ae23a90947120, store: [table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:36,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/d326f5195a0b44b0928ae23a90947120 is 175, key is test_row_0/A:col10/1733814935715/Put/seqid=0 2024-12-10T07:15:36,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742058_1234 (size=39949) 2024-12-10T07:15:36,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:36,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:36,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:36,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:36,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:36,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:36,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:36,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:36,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:36,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:36,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:36,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:36,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:36,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:36,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:36,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814996374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814996374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:36,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:36,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:36,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:36,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:36,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:36,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T07:15:36,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:36,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:36,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:36,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:36,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:36,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:36,557 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=834, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/d326f5195a0b44b0928ae23a90947120 2024-12-10T07:15:36,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/6d9b0e6fa1334349960ff6ec2ef00944 is 50, key is test_row_0/B:col10/1733814935715/Put/seqid=0 2024-12-10T07:15:36,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742059_1235 (size=12301) 2024-12-10T07:15:36,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:36,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:36,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:36,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:36,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:36,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:36,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60464 deadline: 1733814996805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:36,807 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 573523ff7c74144a068617629803d4b6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:15:36,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:36,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:36,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:36,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:36,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:36,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:36,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:36,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:36,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:36,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60498 deadline: 1733814996825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:36,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:36,827 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_2' in 573523ff7c74144a068617629803d4b6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:15:36,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:36,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:36,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:36,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:36,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60466 deadline: 1733814996835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:36,836 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_2' in 573523ff7c74144a068617629803d4b6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:15:36,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:36,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:36,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:36,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:36,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:36,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60470 deadline: 1733814996885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:36,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60488 deadline: 1733814996886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:15:36,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=834 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/6d9b0e6fa1334349960ff6ec2ef00944 2024-12-10T07:15:36,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:36,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:36,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:36,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:36,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:36,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:36,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:36,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/83ad82c0207c43c1a38316bb3ac7ce34 is 50, key is test_row_0/C:col10/1733814935715/Put/seqid=0 2024-12-10T07:15:36,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742060_1236 (size=12301) 2024-12-10T07:15:37,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:37,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:37,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:37,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:37,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:37,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:37,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:37,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:37,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:37,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:37,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:37,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:37,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=834 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/83ad82c0207c43c1a38316bb3ac7ce34 2024-12-10T07:15:37,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/d326f5195a0b44b0928ae23a90947120 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/d326f5195a0b44b0928ae23a90947120 2024-12-10T07:15:37,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/d326f5195a0b44b0928ae23a90947120, entries=200, sequenceid=834, filesize=39.0 K 2024-12-10T07:15:37,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/6d9b0e6fa1334349960ff6ec2ef00944 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/6d9b0e6fa1334349960ff6ec2ef00944 2024-12-10T07:15:37,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/6d9b0e6fa1334349960ff6ec2ef00944, entries=150, sequenceid=834, filesize=12.0 K 2024-12-10T07:15:37,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/83ad82c0207c43c1a38316bb3ac7ce34 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/83ad82c0207c43c1a38316bb3ac7ce34 2024-12-10T07:15:37,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/83ad82c0207c43c1a38316bb3ac7ce34, entries=150, sequenceid=834, filesize=12.0 K 2024-12-10T07:15:37,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 573523ff7c74144a068617629803d4b6 in 1723ms, sequenceid=834, compaction requested=true 2024-12-10T07:15:37,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:37,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:15:37,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:37,440 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:37,440 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:37,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:15:37,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:37,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 573523ff7c74144a068617629803d4b6:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:15:37,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:37,443 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:37,443 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/B is initiating minor compaction (all files) 2024-12-10T07:15:37,443 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/B in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:37,444 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/e54c4fda6e8e40518e6a03966208ad5b, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/2dcaa46d3dc54ed7bf9d7d4f126e434d, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/6d9b0e6fa1334349960ff6ec2ef00944] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=36.9 K 2024-12-10T07:15:37,445 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120763 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:37,445 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/A is initiating minor compaction (all files) 2024-12-10T07:15:37,445 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/A in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:37,446 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/cf9a7f44258847cb9124daee45823573, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/a430374a38df44c5b8f7104361ff88e9, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/d326f5195a0b44b0928ae23a90947120] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=117.9 K 2024-12-10T07:15:37,446 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:37,446 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. files: [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/cf9a7f44258847cb9124daee45823573, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/a430374a38df44c5b8f7104361ff88e9, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/d326f5195a0b44b0928ae23a90947120] 2024-12-10T07:15:37,446 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting e54c4fda6e8e40518e6a03966208ad5b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1733814933062 2024-12-10T07:15:37,449 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf9a7f44258847cb9124daee45823573, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1733814933062 2024-12-10T07:15:37,449 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dcaa46d3dc54ed7bf9d7d4f126e434d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=818, earliestPutTs=1733814934612 2024-12-10T07:15:37,450 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting a430374a38df44c5b8f7104361ff88e9, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=818, earliestPutTs=1733814934580 2024-12-10T07:15:37,450 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d9b0e6fa1334349960ff6ec2ef00944, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=834, earliestPutTs=1733814934657 2024-12-10T07:15:37,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-10T07:15:37,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:37,450 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting d326f5195a0b44b0928ae23a90947120, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=834, earliestPutTs=1733814934657 2024-12-10T07:15:37,451 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 573523ff7c74144a068617629803d4b6 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T07:15:37,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=A 2024-12-10T07:15:37,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:37,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=B 2024-12-10T07:15:37,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:37,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=C 2024-12-10T07:15:37,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:37,476 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#B#compaction#191 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:37,476 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/d6a12949c3204ea193c0b0e515c58161 is 50, key is test_row_0/B:col10/1733814935715/Put/seqid=0 2024-12-10T07:15:37,501 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:37,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210ff993234aa4340a0a2555f1f2644917c_573523ff7c74144a068617629803d4b6 is 50, key is test_row_0/A:col10/1733814935732/Put/seqid=0 2024-12-10T07:15:37,521 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121049b9ca1e502d4bb283833fd0a54b458b_573523ff7c74144a068617629803d4b6 store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:37,545 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121049b9ca1e502d4bb283833fd0a54b458b_573523ff7c74144a068617629803d4b6, store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:37,545 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121049b9ca1e502d4bb283833fd0a54b458b_573523ff7c74144a068617629803d4b6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:37,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742061_1237 (size=13323) 2024-12-10T07:15:37,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742062_1238 (size=12454) 2024-12-10T07:15:37,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:37,567 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210ff993234aa4340a0a2555f1f2644917c_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ff993234aa4340a0a2555f1f2644917c_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:37,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/214484452aae41018da15266f0c58c79, store: [table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:37,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/214484452aae41018da15266f0c58c79 is 175, key is test_row_0/A:col10/1733814935732/Put/seqid=0 2024-12-10T07:15:37,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742063_1239 (size=4469) 2024-12-10T07:15:37,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742064_1240 (size=31255) 2024-12-10T07:15:37,595 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=853, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/214484452aae41018da15266f0c58c79 2024-12-10T07:15:37,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/7ee69754df6649a8b4643b9a0a6008c7 is 50, key is test_row_0/B:col10/1733814935732/Put/seqid=0 2024-12-10T07:15:37,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742065_1241 (size=12301) 2024-12-10T07:15:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:37,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. as already flushing 2024-12-10T07:15:37,901 INFO [Thread-601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-601. 2024-12-10T07:15:37,901 DEBUG [Thread-601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:15:37,901 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:37,901 INFO [Thread-597 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-597. 2024-12-10T07:15:37,901 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:37,901 DEBUG [Thread-597 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:15:37,901 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:37,901 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:37,901 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:15:37,901 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:15:37,980 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#A#compaction#192 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:37,981 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/4278b2bc1be5424eb6b4f5f1eb23bd8e is 175, key is test_row_0/A:col10/1733814935715/Put/seqid=0 2024-12-10T07:15:37,982 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/d6a12949c3204ea193c0b0e515c58161 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/d6a12949c3204ea193c0b0e515c58161 2024-12-10T07:15:37,996 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 573523ff7c74144a068617629803d4b6/B of 573523ff7c74144a068617629803d4b6 into d6a12949c3204ea193c0b0e515c58161(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:37,996 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:37,996 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/B, priority=13, startTime=1733814937440; duration=0sec 2024-12-10T07:15:37,996 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:37,996 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:B 2024-12-10T07:15:37,996 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:37,998 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:37,998 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 573523ff7c74144a068617629803d4b6/C is initiating minor compaction (all files) 2024-12-10T07:15:37,998 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 573523ff7c74144a068617629803d4b6/C in TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:37,998 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/c859d5407dcb44aab6d6e782c67cff30, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/a97e82da1f14476cba2937838740b1e9, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/83ad82c0207c43c1a38316bb3ac7ce34] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp, totalSize=36.9 K 2024-12-10T07:15:37,999 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting c859d5407dcb44aab6d6e782c67cff30, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1733814933062 2024-12-10T07:15:37,999 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting a97e82da1f14476cba2937838740b1e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=818, earliestPutTs=1733814934612 2024-12-10T07:15:38,000 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 83ad82c0207c43c1a38316bb3ac7ce34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=834, earliestPutTs=1733814934657 2024-12-10T07:15:38,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742066_1242 (size=32277) 2024-12-10T07:15:38,045 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 573523ff7c74144a068617629803d4b6#C#compaction#195 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:38,046 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/bc464c7c527b4e5d999aa06525f5d3eb is 50, key is test_row_0/C:col10/1733814935715/Put/seqid=0 2024-12-10T07:15:38,081 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=853 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/7ee69754df6649a8b4643b9a0a6008c7 2024-12-10T07:15:38,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742067_1243 (size=13323) 2024-12-10T07:15:38,117 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/bc464c7c527b4e5d999aa06525f5d3eb as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/bc464c7c527b4e5d999aa06525f5d3eb 2024-12-10T07:15:38,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/1600f92f17644c62a7aa5fd55825c1dd is 50, key is test_row_0/C:col10/1733814935732/Put/seqid=0 2024-12-10T07:15:38,140 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 573523ff7c74144a068617629803d4b6/C of 573523ff7c74144a068617629803d4b6 into bc464c7c527b4e5d999aa06525f5d3eb(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:38,140 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:38,140 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/C, priority=13, startTime=1733814937440; duration=0sec 2024-12-10T07:15:38,140 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:38,140 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:C 2024-12-10T07:15:38,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742068_1244 (size=12301) 2024-12-10T07:15:38,430 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/4278b2bc1be5424eb6b4f5f1eb23bd8e as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/4278b2bc1be5424eb6b4f5f1eb23bd8e 2024-12-10T07:15:38,434 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 573523ff7c74144a068617629803d4b6/A of 573523ff7c74144a068617629803d4b6 into 4278b2bc1be5424eb6b4f5f1eb23bd8e(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:38,434 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:38,435 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., storeName=573523ff7c74144a068617629803d4b6/A, priority=13, startTime=1733814937439; duration=0sec 2024-12-10T07:15:38,435 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:38,435 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 573523ff7c74144a068617629803d4b6:A 2024-12-10T07:15:38,550 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=853 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/1600f92f17644c62a7aa5fd55825c1dd 2024-12-10T07:15:38,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/214484452aae41018da15266f0c58c79 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/214484452aae41018da15266f0c58c79 2024-12-10T07:15:38,566 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/214484452aae41018da15266f0c58c79, entries=150, sequenceid=853, filesize=30.5 K 2024-12-10T07:15:38,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/7ee69754df6649a8b4643b9a0a6008c7 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/7ee69754df6649a8b4643b9a0a6008c7 2024-12-10T07:15:38,572 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/7ee69754df6649a8b4643b9a0a6008c7, entries=150, sequenceid=853, filesize=12.0 K 2024-12-10T07:15:38,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/1600f92f17644c62a7aa5fd55825c1dd as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/1600f92f17644c62a7aa5fd55825c1dd 2024-12-10T07:15:38,576 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/1600f92f17644c62a7aa5fd55825c1dd, entries=150, sequenceid=853, filesize=12.0 K 2024-12-10T07:15:38,577 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=13.42 KB/13740 for 573523ff7c74144a068617629803d4b6 in 1126ms, sequenceid=853, compaction requested=false 2024-12-10T07:15:38,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 573523ff7c74144a068617629803d4b6: 2024-12-10T07:15:38,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:38,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-10T07:15:38,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-10T07:15:38,580 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-10T07:15:38,580 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 6.2250 sec 2024-12-10T07:15:38,582 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 6.2320 sec 2024-12-10T07:15:38,816 INFO [Thread-603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-603. 2024-12-10T07:15:38,816 DEBUG [Thread-603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:15:38,816 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:38,816 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:38,816 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:15:38,846 INFO [Thread-599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-599. 2024-12-10T07:15:38,846 INFO [Thread-605 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-605. 2024-12-10T07:15:38,846 DEBUG [Thread-605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:15:38,846 DEBUG [Thread-599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:15:38,846 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:38,846 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:38,846 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:38,846 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:38,846 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:15:38,846 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:15:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-10T07:15:40,545 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:15:40,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T07:15:40,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 206 2024-12-10T07:15:40,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 123 2024-12-10T07:15:40,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 219 2024-12-10T07:15:40,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 107 2024-12-10T07:15:40,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-12-10T07:15:40,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T07:15:40,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1770 2024-12-10T07:15:40,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1715 2024-12-10T07:15:40,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T07:15:40,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 845 2024-12-10T07:15:40,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2527 rows 2024-12-10T07:15:40,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 852 2024-12-10T07:15:40,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2556 rows 2024-12-10T07:15:40,546 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T07:15:40,546 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T07:15:40,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:40,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:40,546 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:15:40,546 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-10T07:15:40,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T07:15:40,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-10T07:15:40,550 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733814940549"}]},"ts":"1733814940549"} 2024-12-10T07:15:40,551 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T07:15:40,551 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T07:15:40,551 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T07:15:40,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=573523ff7c74144a068617629803d4b6, UNASSIGN}] 2024-12-10T07:15:40,553 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=573523ff7c74144a068617629803d4b6, UNASSIGN 2024-12-10T07:15:40,554 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=573523ff7c74144a068617629803d4b6, regionState=CLOSING, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:40,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=573523ff7c74144a068617629803d4b6, UNASSIGN because future has completed 2024-12-10T07:15:40,555 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T07:15:40,556 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835}] 2024-12-10T07:15:40,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-10T07:15:40,708 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(122): Close 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:40,708 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T07:15:40,708 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1722): Closing 573523ff7c74144a068617629803d4b6, disabling compactions & flushes 2024-12-10T07:15:40,708 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:40,708 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:40,708 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. after waiting 0 ms 2024-12-10T07:15:40,708 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:40,708 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(2902): Flushing 573523ff7c74144a068617629803d4b6 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T07:15:40,709 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=A 2024-12-10T07:15:40,709 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:40,709 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=B 2024-12-10T07:15:40,709 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:40,709 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 573523ff7c74144a068617629803d4b6, store=C 2024-12-10T07:15:40,709 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:40,727 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412106a001007e5b140a9a8f5fe479998b0b4_573523ff7c74144a068617629803d4b6 is 50, key is test_row_1/A:col10/1733814938815/Put/seqid=0 2024-12-10T07:15:40,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742069_1245 (size=9914) 2024-12-10T07:15:40,744 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:40,751 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412106a001007e5b140a9a8f5fe479998b0b4_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412106a001007e5b140a9a8f5fe479998b0b4_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:40,752 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/e62a5e859cc346efa0478af2f938afe9, store: [table=TestAcidGuarantees family=A region=573523ff7c74144a068617629803d4b6] 2024-12-10T07:15:40,752 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/e62a5e859cc346efa0478af2f938afe9 is 175, key is test_row_1/A:col10/1733814938815/Put/seqid=0 2024-12-10T07:15:40,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742070_1246 (size=22561) 2024-12-10T07:15:40,795 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=864, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/e62a5e859cc346efa0478af2f938afe9 2024-12-10T07:15:40,808 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/d1d64c2ff5554c32adeb7170d2e7e955 is 50, key is test_row_1/B:col10/1733814938815/Put/seqid=0 2024-12-10T07:15:40,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742071_1247 (size=9857) 2024-12-10T07:15:40,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-10T07:15:41,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-10T07:15:41,215 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=864 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/d1d64c2ff5554c32adeb7170d2e7e955 2024-12-10T07:15:41,225 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/502c0a1afd93422baad0964c2f1fe3de is 50, key is test_row_1/C:col10/1733814938815/Put/seqid=0 2024-12-10T07:15:41,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742072_1248 (size=9857) 2024-12-10T07:15:41,629 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=864 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/502c0a1afd93422baad0964c2f1fe3de 2024-12-10T07:15:41,633 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/A/e62a5e859cc346efa0478af2f938afe9 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/e62a5e859cc346efa0478af2f938afe9 2024-12-10T07:15:41,637 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/e62a5e859cc346efa0478af2f938afe9, entries=100, sequenceid=864, filesize=22.0 K 2024-12-10T07:15:41,638 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/B/d1d64c2ff5554c32adeb7170d2e7e955 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/d1d64c2ff5554c32adeb7170d2e7e955 2024-12-10T07:15:41,642 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/d1d64c2ff5554c32adeb7170d2e7e955, entries=100, sequenceid=864, filesize=9.6 K 2024-12-10T07:15:41,642 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/.tmp/C/502c0a1afd93422baad0964c2f1fe3de as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/502c0a1afd93422baad0964c2f1fe3de 2024-12-10T07:15:41,647 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/502c0a1afd93422baad0964c2f1fe3de, entries=100, sequenceid=864, filesize=9.6 K 2024-12-10T07:15:41,648 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 573523ff7c74144a068617629803d4b6 in 939ms, sequenceid=864, compaction requested=true 2024-12-10T07:15:41,648 DEBUG [StoreCloser-TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/31e0933d216248ef8cbccd89a28a65e9, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/50dc2eae10534b36bbb1881390225a95, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/b5be2827bea94664a45792772bb536eb, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/026ee71696294d5296102b3e83cf9aa0, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/4ebb58de2647478c9932e270c7b89556, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/726307b9a0a949b89fa754d3ca816921, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/98d123de75304bab8dd88ac359d5e0c2, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/186da5696d7f4a80b7e46bede4321e12, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/2df345b96c7d464f9698e97165d72840, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/fbfe75bf23a149ebb94a11586d8e1637, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/bf846d28d18f46799b35859122ff3379, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/ffaf96053ddd49e3b7c70305ed4dafa1, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/08ad69f7ebd14dd1aba06d268acb548c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/884d943af6f943319c49b3baccc311fc, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/833ffdadaec44481adf26c4481272a55, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/fd740df15e0f4620a9e0bd1e11a9b0ba, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/eea4b1090ec048839820ae465872d500, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/bfa3b21ede7640479e10f40ae22940ee, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/ba3290cd8cf7425c8cc5967c7e1352a9, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/7672041d67fc42c8aaba5481403002b1, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/576aede2cf5a439a873ba639bfe421d5, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/c35fd6ec90924993a41ed932e33885b5, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/29b8e59e0094438da460a16cd6f13109, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/1ca04cefafc7437e903a5d8efbc4a78c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/04b8ad20c2e94ed1be12e579b576d745, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/3f956f4adfa047a29f991c4267e9b913, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/3a36e44d8a4f45e8a1c37881236343db, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/cf9a7f44258847cb9124daee45823573, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/a430374a38df44c5b8f7104361ff88e9, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/d326f5195a0b44b0928ae23a90947120] to archive 2024-12-10T07:15:41,649 DEBUG [StoreCloser-TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T07:15:41,653 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/31e0933d216248ef8cbccd89a28a65e9 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/31e0933d216248ef8cbccd89a28a65e9 2024-12-10T07:15:41,653 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/50dc2eae10534b36bbb1881390225a95 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/50dc2eae10534b36bbb1881390225a95 2024-12-10T07:15:41,653 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/b5be2827bea94664a45792772bb536eb to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/b5be2827bea94664a45792772bb536eb 2024-12-10T07:15:41,653 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/726307b9a0a949b89fa754d3ca816921 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/726307b9a0a949b89fa754d3ca816921 2024-12-10T07:15:41,654 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/026ee71696294d5296102b3e83cf9aa0 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/026ee71696294d5296102b3e83cf9aa0 2024-12-10T07:15:41,654 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/4ebb58de2647478c9932e270c7b89556 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/4ebb58de2647478c9932e270c7b89556 2024-12-10T07:15:41,654 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/98d123de75304bab8dd88ac359d5e0c2 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/98d123de75304bab8dd88ac359d5e0c2 2024-12-10T07:15:41,654 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/186da5696d7f4a80b7e46bede4321e12 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/186da5696d7f4a80b7e46bede4321e12 2024-12-10T07:15:41,656 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/2df345b96c7d464f9698e97165d72840 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/2df345b96c7d464f9698e97165d72840 2024-12-10T07:15:41,656 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/fbfe75bf23a149ebb94a11586d8e1637 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/fbfe75bf23a149ebb94a11586d8e1637 2024-12-10T07:15:41,656 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/bf846d28d18f46799b35859122ff3379 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/bf846d28d18f46799b35859122ff3379 2024-12-10T07:15:41,657 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/ffaf96053ddd49e3b7c70305ed4dafa1 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/ffaf96053ddd49e3b7c70305ed4dafa1 2024-12-10T07:15:41,657 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/08ad69f7ebd14dd1aba06d268acb548c to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/08ad69f7ebd14dd1aba06d268acb548c 2024-12-10T07:15:41,658 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/fd740df15e0f4620a9e0bd1e11a9b0ba to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/fd740df15e0f4620a9e0bd1e11a9b0ba 2024-12-10T07:15:41,658 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/833ffdadaec44481adf26c4481272a55 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/833ffdadaec44481adf26c4481272a55 2024-12-10T07:15:41,658 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/884d943af6f943319c49b3baccc311fc to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/884d943af6f943319c49b3baccc311fc 2024-12-10T07:15:41,659 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/eea4b1090ec048839820ae465872d500 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/eea4b1090ec048839820ae465872d500 2024-12-10T07:15:41,659 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/7672041d67fc42c8aaba5481403002b1 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/7672041d67fc42c8aaba5481403002b1 2024-12-10T07:15:41,659 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/ba3290cd8cf7425c8cc5967c7e1352a9 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/ba3290cd8cf7425c8cc5967c7e1352a9 2024-12-10T07:15:41,659 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/bfa3b21ede7640479e10f40ae22940ee to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/bfa3b21ede7640479e10f40ae22940ee 2024-12-10T07:15:41,661 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/576aede2cf5a439a873ba639bfe421d5 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/576aede2cf5a439a873ba639bfe421d5 2024-12-10T07:15:41,661 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/c35fd6ec90924993a41ed932e33885b5 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/c35fd6ec90924993a41ed932e33885b5 2024-12-10T07:15:41,661 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/29b8e59e0094438da460a16cd6f13109 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/29b8e59e0094438da460a16cd6f13109 2024-12-10T07:15:41,669 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/cf9a7f44258847cb9124daee45823573 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/cf9a7f44258847cb9124daee45823573 2024-12-10T07:15:41,669 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/1ca04cefafc7437e903a5d8efbc4a78c to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/1ca04cefafc7437e903a5d8efbc4a78c 2024-12-10T07:15:41,670 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/04b8ad20c2e94ed1be12e579b576d745 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/04b8ad20c2e94ed1be12e579b576d745 2024-12-10T07:15:41,670 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/3a36e44d8a4f45e8a1c37881236343db to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/3a36e44d8a4f45e8a1c37881236343db 2024-12-10T07:15:41,671 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/3f956f4adfa047a29f991c4267e9b913 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/3f956f4adfa047a29f991c4267e9b913 2024-12-10T07:15:41,671 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/d326f5195a0b44b0928ae23a90947120 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/d326f5195a0b44b0928ae23a90947120 2024-12-10T07:15:41,671 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/a430374a38df44c5b8f7104361ff88e9 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/a430374a38df44c5b8f7104361ff88e9 2024-12-10T07:15:41,673 DEBUG [StoreCloser-TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/4f639267d7a24e9380441020435d4ca0, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/16b23a711b8545c6aa4891c53aa5d204, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/e458cac9d6314d73961bfb4553c49690, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/7d3030b917724373af48e86c76f150d0, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/aa3ba355083341af812c630ca221d345, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/95afc3610c7b446bac6fe89ea3c60df6, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/b191e2d24d814d64a58899c277b4b604, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/c3a759b156394c8ca3ae6bde57dd9fc9, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/31f2a2244ad94369a657038f285b37ba, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/5bc2beaa1cae48f29cfd8b5e6c6b8fd6, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/1f4f8990b10f491b9dbfeb9678923168, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/f24922ff033249f183b7193cc54e68ff, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/fd9ed32eb9c04dc098c9bcbc89a3de2f, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/41eb4ec95803460cac59824b7c57aa79, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/0a065ab0d35d4bcb930075c4dbc396fd, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/c8f4a7784869406689dc2ae993d56c14, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/5a507d6eb0d64a7499f5f242515591b8, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/c7e02b0de145415c830acfd9be72bdbb, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/030fb5dbf2e84cd094e4cb216813fd3b, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/9db61db519dc4834b4fc42e6ef0f6c89, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/7f082eb1c8d04d1ab16e2dbb778993ff, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/0235d077e67d44aaba2bccbf8d56eea9, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/c847e2c54ffe430e82d6001462a0bd9c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/bc5eff2ac15b4750ad39c19a6be65f43, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/87fd7489c19e45ed9d68180e76499b42, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/a16d5ea619f342af9c3e5e5d5fd1a2a6, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/e54c4fda6e8e40518e6a03966208ad5b, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/8175457833284fe8a2a1ea20e006c8ec, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/2dcaa46d3dc54ed7bf9d7d4f126e434d, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/6d9b0e6fa1334349960ff6ec2ef00944] to archive 2024-12-10T07:15:41,674 DEBUG [StoreCloser-TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T07:15:41,678 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/16b23a711b8545c6aa4891c53aa5d204 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/16b23a711b8545c6aa4891c53aa5d204 2024-12-10T07:15:41,681 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/7d3030b917724373af48e86c76f150d0 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/7d3030b917724373af48e86c76f150d0 2024-12-10T07:15:41,681 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/95afc3610c7b446bac6fe89ea3c60df6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/95afc3610c7b446bac6fe89ea3c60df6 2024-12-10T07:15:41,681 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/e458cac9d6314d73961bfb4553c49690 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/e458cac9d6314d73961bfb4553c49690 2024-12-10T07:15:41,681 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/4f639267d7a24e9380441020435d4ca0 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/4f639267d7a24e9380441020435d4ca0 2024-12-10T07:15:41,681 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/aa3ba355083341af812c630ca221d345 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/aa3ba355083341af812c630ca221d345 2024-12-10T07:15:41,683 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/c3a759b156394c8ca3ae6bde57dd9fc9 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/c3a759b156394c8ca3ae6bde57dd9fc9 2024-12-10T07:15:41,683 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/b191e2d24d814d64a58899c277b4b604 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/b191e2d24d814d64a58899c277b4b604 2024-12-10T07:15:41,684 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/31f2a2244ad94369a657038f285b37ba to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/31f2a2244ad94369a657038f285b37ba 2024-12-10T07:15:41,684 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/5bc2beaa1cae48f29cfd8b5e6c6b8fd6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/5bc2beaa1cae48f29cfd8b5e6c6b8fd6 2024-12-10T07:15:41,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-10T07:15:41,685 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/41eb4ec95803460cac59824b7c57aa79 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/41eb4ec95803460cac59824b7c57aa79 2024-12-10T07:15:41,685 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/f24922ff033249f183b7193cc54e68ff to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/f24922ff033249f183b7193cc54e68ff 2024-12-10T07:15:41,685 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/fd9ed32eb9c04dc098c9bcbc89a3de2f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/fd9ed32eb9c04dc098c9bcbc89a3de2f 2024-12-10T07:15:41,685 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/1f4f8990b10f491b9dbfeb9678923168 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/1f4f8990b10f491b9dbfeb9678923168 2024-12-10T07:15:41,686 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/c8f4a7784869406689dc2ae993d56c14 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/c8f4a7784869406689dc2ae993d56c14 2024-12-10T07:15:41,686 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/0a065ab0d35d4bcb930075c4dbc396fd to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/0a065ab0d35d4bcb930075c4dbc396fd 2024-12-10T07:15:41,688 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/c7e02b0de145415c830acfd9be72bdbb to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/c7e02b0de145415c830acfd9be72bdbb 2024-12-10T07:15:41,688 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/5a507d6eb0d64a7499f5f242515591b8 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/5a507d6eb0d64a7499f5f242515591b8 2024-12-10T07:15:41,688 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/9db61db519dc4834b4fc42e6ef0f6c89 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/9db61db519dc4834b4fc42e6ef0f6c89 2024-12-10T07:15:41,689 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/c847e2c54ffe430e82d6001462a0bd9c to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/c847e2c54ffe430e82d6001462a0bd9c 2024-12-10T07:15:41,689 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/030fb5dbf2e84cd094e4cb216813fd3b to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/030fb5dbf2e84cd094e4cb216813fd3b 2024-12-10T07:15:41,689 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/7f082eb1c8d04d1ab16e2dbb778993ff to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/7f082eb1c8d04d1ab16e2dbb778993ff 2024-12-10T07:15:41,689 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/bc5eff2ac15b4750ad39c19a6be65f43 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/bc5eff2ac15b4750ad39c19a6be65f43 2024-12-10T07:15:41,689 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/0235d077e67d44aaba2bccbf8d56eea9 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/0235d077e67d44aaba2bccbf8d56eea9 2024-12-10T07:15:41,693 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/e54c4fda6e8e40518e6a03966208ad5b to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/e54c4fda6e8e40518e6a03966208ad5b 2024-12-10T07:15:41,693 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/a16d5ea619f342af9c3e5e5d5fd1a2a6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/a16d5ea619f342af9c3e5e5d5fd1a2a6 2024-12-10T07:15:41,694 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/87fd7489c19e45ed9d68180e76499b42 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/87fd7489c19e45ed9d68180e76499b42 2024-12-10T07:15:41,696 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/6d9b0e6fa1334349960ff6ec2ef00944 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/6d9b0e6fa1334349960ff6ec2ef00944 2024-12-10T07:15:41,696 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/8175457833284fe8a2a1ea20e006c8ec to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/8175457833284fe8a2a1ea20e006c8ec 2024-12-10T07:15:41,696 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/2dcaa46d3dc54ed7bf9d7d4f126e434d to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/2dcaa46d3dc54ed7bf9d7d4f126e434d 2024-12-10T07:15:41,698 DEBUG [StoreCloser-TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/71360b8471704734a8a36a1bc822f79c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/af68ca007b374c119f90cb4685a9655e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/4a2c71334fa842f3ab7c3aed31032327, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/6660049533a64aa8b1b5778dedf06432, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/72b64c4999a145e9921a7cc8ca816f25, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/841cf0687b664325b388db317586f8b1, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/4bd41cd1d6b642a5be64938a60df95c2, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/e39cd111f67c4d098a6f152312d43cbb, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/42468d3db4c14c1da24ccad3fd435f95, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/ec4ec1fda08a4fdba8747fe10bf91f54, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/d75b15992ea441a1ba487ef58afc33f0, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/e8ed2676de8b4e46a336b16674b382bb, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/72c7305da8ed424394b9b72b16f77781, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/97b53732d69e4269b6577ad6ff150de3, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/4ba6dbfa77d945c389a3fb8f17d8158a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/f4cbafda7d3c4fadb6e0c7db101c4ab0, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/36ae73aeaccd495fb7e14afee0afc470, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/48adc0497b934849bf23a7e58ee60046, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/2a03d932a89f4a98b2cd6c995e5143e5, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/2e0a6b11cc054fec9619438fc2df139d, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/a213e3bfbc764592a946818b746fd390, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/47066829368b4c6a8568d3c8e79497c1, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/d717ecdd57d04ef093a64293a6825478, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/600fef4dc9f54bb8854ec133a95720f4, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/02a182e187094a08b13a9a4684c5dde4, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/2f6487d9109c4b8e9a5e877c305c4e0a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/c859d5407dcb44aab6d6e782c67cff30, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/d51f19e13e6b42d18b8e51e9020e4f88, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/a97e82da1f14476cba2937838740b1e9, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/83ad82c0207c43c1a38316bb3ac7ce34] to archive 2024-12-10T07:15:41,700 DEBUG [StoreCloser-TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T07:15:41,710 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/71360b8471704734a8a36a1bc822f79c to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/71360b8471704734a8a36a1bc822f79c 2024-12-10T07:15:41,710 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/72b64c4999a145e9921a7cc8ca816f25 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/72b64c4999a145e9921a7cc8ca816f25 2024-12-10T07:15:41,710 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/6660049533a64aa8b1b5778dedf06432 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/6660049533a64aa8b1b5778dedf06432 2024-12-10T07:15:41,710 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/4a2c71334fa842f3ab7c3aed31032327 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/4a2c71334fa842f3ab7c3aed31032327 2024-12-10T07:15:41,710 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/af68ca007b374c119f90cb4685a9655e to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/af68ca007b374c119f90cb4685a9655e 2024-12-10T07:15:41,710 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/841cf0687b664325b388db317586f8b1 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/841cf0687b664325b388db317586f8b1 2024-12-10T07:15:41,711 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/e39cd111f67c4d098a6f152312d43cbb to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/e39cd111f67c4d098a6f152312d43cbb 2024-12-10T07:15:41,711 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/4bd41cd1d6b642a5be64938a60df95c2 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/4bd41cd1d6b642a5be64938a60df95c2 2024-12-10T07:15:41,717 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/ec4ec1fda08a4fdba8747fe10bf91f54 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/ec4ec1fda08a4fdba8747fe10bf91f54 2024-12-10T07:15:41,719 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/d75b15992ea441a1ba487ef58afc33f0 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/d75b15992ea441a1ba487ef58afc33f0 2024-12-10T07:15:41,719 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/e8ed2676de8b4e46a336b16674b382bb to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/e8ed2676de8b4e46a336b16674b382bb 2024-12-10T07:15:41,719 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/72c7305da8ed424394b9b72b16f77781 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/72c7305da8ed424394b9b72b16f77781 2024-12-10T07:15:41,719 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/f4cbafda7d3c4fadb6e0c7db101c4ab0 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/f4cbafda7d3c4fadb6e0c7db101c4ab0 2024-12-10T07:15:41,719 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/97b53732d69e4269b6577ad6ff150de3 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/97b53732d69e4269b6577ad6ff150de3 2024-12-10T07:15:41,719 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/4ba6dbfa77d945c389a3fb8f17d8158a to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/4ba6dbfa77d945c389a3fb8f17d8158a 2024-12-10T07:15:41,720 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/36ae73aeaccd495fb7e14afee0afc470 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/36ae73aeaccd495fb7e14afee0afc470 2024-12-10T07:15:41,721 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/42468d3db4c14c1da24ccad3fd435f95 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/42468d3db4c14c1da24ccad3fd435f95 2024-12-10T07:15:41,722 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/2a03d932a89f4a98b2cd6c995e5143e5 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/2a03d932a89f4a98b2cd6c995e5143e5 2024-12-10T07:15:41,722 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/48adc0497b934849bf23a7e58ee60046 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/48adc0497b934849bf23a7e58ee60046 2024-12-10T07:15:41,723 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/2e0a6b11cc054fec9619438fc2df139d to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/2e0a6b11cc054fec9619438fc2df139d 2024-12-10T07:15:41,723 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/47066829368b4c6a8568d3c8e79497c1 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/47066829368b4c6a8568d3c8e79497c1 2024-12-10T07:15:41,723 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/a213e3bfbc764592a946818b746fd390 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/a213e3bfbc764592a946818b746fd390 2024-12-10T07:15:41,723 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/d717ecdd57d04ef093a64293a6825478 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/d717ecdd57d04ef093a64293a6825478 2024-12-10T07:15:41,724 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/600fef4dc9f54bb8854ec133a95720f4 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/600fef4dc9f54bb8854ec133a95720f4 2024-12-10T07:15:41,724 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/02a182e187094a08b13a9a4684c5dde4 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/02a182e187094a08b13a9a4684c5dde4 2024-12-10T07:15:41,724 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/c859d5407dcb44aab6d6e782c67cff30 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/c859d5407dcb44aab6d6e782c67cff30 2024-12-10T07:15:41,724 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/2f6487d9109c4b8e9a5e877c305c4e0a to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/2f6487d9109c4b8e9a5e877c305c4e0a 2024-12-10T07:15:41,729 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/d51f19e13e6b42d18b8e51e9020e4f88 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/d51f19e13e6b42d18b8e51e9020e4f88 2024-12-10T07:15:41,729 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/83ad82c0207c43c1a38316bb3ac7ce34 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/83ad82c0207c43c1a38316bb3ac7ce34 2024-12-10T07:15:41,729 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/a97e82da1f14476cba2937838740b1e9 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/a97e82da1f14476cba2937838740b1e9 2024-12-10T07:15:41,734 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/recovered.edits/867.seqid, newMaxSeqId=867, maxSeqId=4 2024-12-10T07:15:41,735 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. 2024-12-10T07:15:41,735 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1676): Region close journal for 573523ff7c74144a068617629803d4b6: Waiting for close lock at 1733814940708Running coprocessor pre-close hooks at 1733814940708Disabling compacts and flushes for region at 1733814940708Disabling writes for close at 1733814940708Obtaining lock to block concurrent updates at 1733814940708Preparing flush snapshotting stores in 573523ff7c74144a068617629803d4b6 at 1733814940708Finished memstore snapshotting TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733814940709 (+1 ms)Flushing stores of TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6. at 1733814940709Flushing 573523ff7c74144a068617629803d4b6/A: creating writer at 1733814940710 (+1 ms)Flushing mob file 573523ff7c74144a068617629803d4b6/A: appending metadata at 1733814940727 (+17 ms)Flushing mob file 573523ff7c74144a068617629803d4b6/A: closing flushed file at 1733814940727Flushing 573523ff7c74144a068617629803d4b6/A: appending metadata at 1733814940752 (+25 ms)Flushing 573523ff7c74144a068617629803d4b6/A: closing flushed file at 1733814940752Flushing 573523ff7c74144a068617629803d4b6/B: creating writer at 1733814940802 (+50 ms)Flushing 573523ff7c74144a068617629803d4b6/B: appending metadata at 1733814940807 (+5 ms)Flushing 573523ff7c74144a068617629803d4b6/B: closing flushed file at 1733814940807Flushing 573523ff7c74144a068617629803d4b6/C: creating writer at 1733814941220 (+413 ms)Flushing 573523ff7c74144a068617629803d4b6/C: appending metadata at 1733814941225 (+5 ms)Flushing 573523ff7c74144a068617629803d4b6/C: closing flushed file at 1733814941225Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f770cd0: reopening flushed file at 1733814941633 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ba781cd: reopening flushed file at 1733814941637 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58580551: reopening flushed file at 1733814941642 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 573523ff7c74144a068617629803d4b6 in 939ms, sequenceid=864, compaction requested=true at 1733814941648 (+6 ms)Writing region close event to WAL at 1733814941731 (+83 ms)Running coprocessor post-close hooks at 1733814941735 (+4 ms)Closed at 1733814941735 2024-12-10T07:15:41,737 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(157): Closed 573523ff7c74144a068617629803d4b6 2024-12-10T07:15:41,737 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=573523ff7c74144a068617629803d4b6, regionState=CLOSED 2024-12-10T07:15:41,739 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 because future has completed 2024-12-10T07:15:41,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-10T07:15:41,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseRegionProcedure 573523ff7c74144a068617629803d4b6, server=b1396b8d2d12,35871,1733814875835 in 1.1850 sec 2024-12-10T07:15:41,743 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-12-10T07:15:41,743 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=573523ff7c74144a068617629803d4b6, UNASSIGN in 1.1880 sec 2024-12-10T07:15:41,744 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-10T07:15:41,744 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1920 sec 2024-12-10T07:15:41,745 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733814941745"}]},"ts":"1733814941745"} 2024-12-10T07:15:41,747 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T07:15:41,747 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T07:15:41,748 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.2010 sec 2024-12-10T07:15:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-10T07:15:42,695 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:15:42,696 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-10T07:15:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:15:42,697 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:15:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-10T07:15:42,698 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=54, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:15:42,700 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6 2024-12-10T07:15:42,702 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A, FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B, FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C, FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/recovered.edits] 2024-12-10T07:15:42,705 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/e62a5e859cc346efa0478af2f938afe9 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/e62a5e859cc346efa0478af2f938afe9 2024-12-10T07:15:42,705 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/4278b2bc1be5424eb6b4f5f1eb23bd8e to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/4278b2bc1be5424eb6b4f5f1eb23bd8e 2024-12-10T07:15:42,705 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/214484452aae41018da15266f0c58c79 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/A/214484452aae41018da15266f0c58c79 2024-12-10T07:15:42,708 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/d6a12949c3204ea193c0b0e515c58161 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/d6a12949c3204ea193c0b0e515c58161 2024-12-10T07:15:42,708 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/d1d64c2ff5554c32adeb7170d2e7e955 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/d1d64c2ff5554c32adeb7170d2e7e955 2024-12-10T07:15:42,708 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/7ee69754df6649a8b4643b9a0a6008c7 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/B/7ee69754df6649a8b4643b9a0a6008c7 2024-12-10T07:15:42,711 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/1600f92f17644c62a7aa5fd55825c1dd to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/1600f92f17644c62a7aa5fd55825c1dd 2024-12-10T07:15:42,711 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/502c0a1afd93422baad0964c2f1fe3de to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/502c0a1afd93422baad0964c2f1fe3de 2024-12-10T07:15:42,711 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/bc464c7c527b4e5d999aa06525f5d3eb to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/C/bc464c7c527b4e5d999aa06525f5d3eb 2024-12-10T07:15:42,714 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/recovered.edits/867.seqid to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6/recovered.edits/867.seqid 2024-12-10T07:15:42,715 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/573523ff7c74144a068617629803d4b6 2024-12-10T07:15:42,715 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T07:15:42,715 DEBUG [PEWorker-3 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T07:15:42,716 DEBUG [PEWorker-3 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-10T07:15:42,726 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210306d03b98d0f495a8df5d10aeeac4db9_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210306d03b98d0f495a8df5d10aeeac4db9_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:42,743 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121037caa257f8064e2999072db0f715ce31_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121037caa257f8064e2999072db0f715ce31_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:42,743 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210254a85fb940d43b4a257248a799c3fcb_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210254a85fb940d43b4a257248a799c3fcb_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:42,743 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103d949a8db9ca44de9a7a1e6eaed3ef95_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412103d949a8db9ca44de9a7a1e6eaed3ef95_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:42,743 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121024b327912e15420580a4ad5f773d2e81_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121024b327912e15420580a4ad5f773d2e81_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:42,744 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121038c2eafbf08548c0b36eb63bbfb124f3_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121038c2eafbf08548c0b36eb63bbfb124f3_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:42,744 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121053d0d9562efb4b0e8faf5e333da44cc8_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121053d0d9562efb4b0e8faf5e333da44cc8_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:42,744 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121030d6a3e817794299b0ece62d1300f8e6_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121030d6a3e817794299b0ece62d1300f8e6_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:42,745 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412106a001007e5b140a9a8f5fe479998b0b4_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412106a001007e5b140a9a8f5fe479998b0b4_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:42,746 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105aa2dfc9a70a42feb2e4b925200141c2_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105aa2dfc9a70a42feb2e4b925200141c2_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:42,746 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121064e3accd2da94e209605f0c76d77c8ce_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121064e3accd2da94e209605f0c76d77c8ce_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:42,746 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121084e3406923704a7f80908b1e6734b8b8_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121084e3406923704a7f80908b1e6734b8b8_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:42,746 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412107447ac47e74c42578ed77393bb65cfcb_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412107447ac47e74c42578ed77393bb65cfcb_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:42,746 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210969ef5dbba4d4a3086077ff524de909b_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210969ef5dbba4d4a3086077ff524de909b_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:42,747 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108e06f8bca1e44716b72e4538ba296ac6_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412108e06f8bca1e44716b72e4538ba296ac6_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:42,747 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210980cd7d7fdd54a3199595841826c74fb_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210980cd7d7fdd54a3199595841826c74fb_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:42,748 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a78dfea7e5a94436952763fff64ad364_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a78dfea7e5a94436952763fff64ad364_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:42,748 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b1b61e4165e647e28dc86307030b1d50_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210b1b61e4165e647e28dc86307030b1d50_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:42,748 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ccbc0d625c234f25b53d7986d876f767_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ccbc0d625c234f25b53d7986d876f767_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:42,749 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ddf7022632c34588a31da5b85b5cef27_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ddf7022632c34588a31da5b85b5cef27_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:42,749 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210e1491ba1cb8e4011bde07d94ea323b7d_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210e1491ba1cb8e4011bde07d94ea323b7d_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:42,749 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ef5098068af2448fa2687e4cf3b1622a_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ef5098068af2448fa2687e4cf3b1622a_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:42,749 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ff993234aa4340a0a2555f1f2644917c_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ff993234aa4340a0a2555f1f2644917c_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:42,749 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f2cc2dac52574a168408300e1ba245ec_573523ff7c74144a068617629803d4b6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f2cc2dac52574a168408300e1ba245ec_573523ff7c74144a068617629803d4b6 2024-12-10T07:15:42,750 DEBUG [PEWorker-3 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T07:15:42,754 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=54, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:15:42,758 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T07:15:42,773 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T07:15:42,781 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=54, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:15:42,781 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T07:15:42,781 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733814942781"}]},"ts":"9223372036854775807"} 2024-12-10T07:15:42,785 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T07:15:42,785 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 573523ff7c74144a068617629803d4b6, NAME => 'TestAcidGuarantees,,1733814912762.573523ff7c74144a068617629803d4b6.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T07:15:42,785 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T07:15:42,785 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733814942785"}]},"ts":"9223372036854775807"} 2024-12-10T07:15:42,793 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T07:15:42,794 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=54, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:15:42,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 98 msec 2024-12-10T07:15:42,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-10T07:15:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T07:15:42,806 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:15:42,819 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=258 (was 250) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/cluster_ac824e32-1cbb-51f2-a594-edd33703f97a/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-673173346_22 at /127.0.0.1:40984 [Waiting for operation #443] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-673173346_22 at /127.0.0.1:47930 [Waiting for operation #311] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1814971026_22 at /127.0.0.1:41084 [Waiting for operation #409] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/cluster_ac824e32-1cbb-51f2-a594-edd33703f97a/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1814971026_22 at /127.0.0.1:47938 [Waiting for operation #295] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=463 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=957 (was 829) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3999 (was 3403) - AvailableMemoryMB LEAK? - 2024-12-10T07:15:42,843 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=258, OpenFileDescriptor=465, MaxFileDescriptor=1048576, SystemLoadAverage=957, ProcessCount=11, AvailableMemoryMB=3998 2024-12-10T07:15:42,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T07:15:42,845 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T07:15:42,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T07:15:42,851 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T07:15:42,852 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:42,852 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 55 2024-12-10T07:15:42,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-10T07:15:42,854 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T07:15:42,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742073_1249 (size=1061) 2024-12-10T07:15:42,905 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8 2024-12-10T07:15:42,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742074_1250 (size=53) 2024-12-10T07:15:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-10T07:15:43,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-10T07:15:43,353 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T07:15:43,353 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 07d0f6bb02ffe54509ba6124b6830098, disabling compactions & flushes 2024-12-10T07:15:43,353 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:43,353 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:43,353 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. after waiting 0 ms 2024-12-10T07:15:43,353 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:43,353 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:43,353 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 07d0f6bb02ffe54509ba6124b6830098: Waiting for close lock at 1733814943353Disabling compacts and flushes for region at 1733814943353Disabling writes for close at 1733814943353Writing region close event to WAL at 1733814943353Closed at 1733814943353 2024-12-10T07:15:43,354 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T07:15:43,354 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733814943354"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733814943354"}]},"ts":"1733814943354"} 2024-12-10T07:15:43,356 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T07:15:43,357 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T07:15:43,357 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733814943357"}]},"ts":"1733814943357"} 2024-12-10T07:15:43,361 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T07:15:43,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=07d0f6bb02ffe54509ba6124b6830098, ASSIGN}] 2024-12-10T07:15:43,363 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=07d0f6bb02ffe54509ba6124b6830098, ASSIGN 2024-12-10T07:15:43,363 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=07d0f6bb02ffe54509ba6124b6830098, ASSIGN; state=OFFLINE, location=b1396b8d2d12,35871,1733814875835; forceNewPlan=false, retain=false 2024-12-10T07:15:43,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-10T07:15:43,514 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=07d0f6bb02ffe54509ba6124b6830098, regionState=OPENING, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:43,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=07d0f6bb02ffe54509ba6124b6830098, ASSIGN because future has completed 2024-12-10T07:15:43,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835}] 2024-12-10T07:15:43,670 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:43,670 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7752): Opening region: {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} 2024-12-10T07:15:43,670 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:43,671 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T07:15:43,671 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7794): checking encryption for 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:43,671 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7797): checking classloading for 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:43,672 INFO [StoreOpener-07d0f6bb02ffe54509ba6124b6830098-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:43,673 INFO [StoreOpener-07d0f6bb02ffe54509ba6124b6830098-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:15:43,673 INFO [StoreOpener-07d0f6bb02ffe54509ba6124b6830098-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07d0f6bb02ffe54509ba6124b6830098 columnFamilyName A 2024-12-10T07:15:43,673 DEBUG [StoreOpener-07d0f6bb02ffe54509ba6124b6830098-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:43,673 INFO [StoreOpener-07d0f6bb02ffe54509ba6124b6830098-1 {}] regionserver.HStore(327): Store=07d0f6bb02ffe54509ba6124b6830098/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:15:43,674 INFO [StoreOpener-07d0f6bb02ffe54509ba6124b6830098-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:43,674 INFO [StoreOpener-07d0f6bb02ffe54509ba6124b6830098-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:15:43,675 INFO [StoreOpener-07d0f6bb02ffe54509ba6124b6830098-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07d0f6bb02ffe54509ba6124b6830098 columnFamilyName B 2024-12-10T07:15:43,675 DEBUG [StoreOpener-07d0f6bb02ffe54509ba6124b6830098-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:43,675 INFO [StoreOpener-07d0f6bb02ffe54509ba6124b6830098-1 {}] regionserver.HStore(327): Store=07d0f6bb02ffe54509ba6124b6830098/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:15:43,675 INFO [StoreOpener-07d0f6bb02ffe54509ba6124b6830098-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:43,676 INFO [StoreOpener-07d0f6bb02ffe54509ba6124b6830098-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:15:43,676 INFO [StoreOpener-07d0f6bb02ffe54509ba6124b6830098-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07d0f6bb02ffe54509ba6124b6830098 columnFamilyName C 2024-12-10T07:15:43,676 DEBUG [StoreOpener-07d0f6bb02ffe54509ba6124b6830098-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:15:43,676 INFO [StoreOpener-07d0f6bb02ffe54509ba6124b6830098-1 {}] regionserver.HStore(327): Store=07d0f6bb02ffe54509ba6124b6830098/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:15:43,677 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:43,677 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1038): replaying wal for 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:43,677 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:43,678 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:43,679 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1048): stopping wal replay for 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:43,679 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1060): Cleaning up temporary data for 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:43,679 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T07:15:43,680 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1093): writing seq id for 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:43,689 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T07:15:43,690 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1114): Opened 07d0f6bb02ffe54509ba6124b6830098; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73077636, jitterRate=0.08894163370132446}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T07:15:43,690 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:43,691 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1006): Region open journal for 07d0f6bb02ffe54509ba6124b6830098: Running coprocessor pre-open hook at 1733814943671Writing region info on filesystem at 1733814943671Initializing all the Stores at 1733814943671Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733814943671Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733814943672 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733814943672Cleaning up temporary data from old regions at 1733814943679 (+7 ms)Running coprocessor post-open hooks at 1733814943690 (+11 ms)Region opened successfully at 1733814943691 (+1 ms) 2024-12-10T07:15:43,692 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., pid=57, masterSystemTime=1733814943667 2024-12-10T07:15:43,693 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:43,693 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:43,694 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=07d0f6bb02ffe54509ba6124b6830098, regionState=OPEN, openSeqNum=2, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:43,696 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 because future has completed 2024-12-10T07:15:43,699 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-10T07:15:43,699 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; OpenRegionProcedure 07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 in 180 msec 2024-12-10T07:15:43,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-12-10T07:15:43,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=07d0f6bb02ffe54509ba6124b6830098, ASSIGN in 337 msec 2024-12-10T07:15:43,702 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T07:15:43,702 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733814943702"}]},"ts":"1733814943702"} 2024-12-10T07:15:43,704 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T07:15:43,705 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T07:15:43,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 860 msec 2024-12-10T07:15:43,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-10T07:15:43,985 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:15:43,986 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43dfb4b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:43,986 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:15:43,986 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:15:43,987 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:15:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:15:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:15:43,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2727e326, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:43,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:15:43,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:15:43,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:43,989 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:15:43,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10cd2de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:43,989 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:15:43,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:15:43,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:15:43,991 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:15:43,992 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T07:15:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is b1396b8d2d12,36259,1733814874775 2024-12-10T07:15:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3fdca488 2024-12-10T07:15:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T07:15:43,994 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T07:15:43,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@634f1deb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:43,995 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:15:43,995 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:15:43,996 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:15:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:15:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:15:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c6fd91e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:15:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:15:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:43,997 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:15:43,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f5bcc33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:43,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32865d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:43,998 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:15:43,998 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:15:43,999 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:15:43,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:15:43,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:15:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d02280e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:15:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:15:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:44,000 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:15:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19f3d28e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:44,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67136d23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:44,002 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:15:44,002 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:15:44,002 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:15:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:15:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:15:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57e3c9e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:15:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:15:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:44,004 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:15:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ac38a85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:44,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@447061ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:44,005 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:15:44,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:15:44,006 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:15:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:15:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:15:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18c483c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:15:44,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:15:44,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:44,008 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:15:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fa30957, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:44,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2847dac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:44,017 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:15:44,017 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:15:44,018 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:15:44,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:15:44,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:15:44,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@da9862a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:44,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:15:44,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:15:44,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:44,020 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:15:44,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@951deb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:44,021 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43f5f704, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:44,021 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:15:44,021 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:15:44,022 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:15:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:15:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:15:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a39cbbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:15:44,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:15:44,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:44,023 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:15:44,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67765f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:44,025 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37ad5c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:44,025 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:15:44,025 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:15:44,025 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:15:44,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:15:44,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:15:44,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20713c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:44,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:15:44,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:15:44,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:44,027 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:15:44,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a98a406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:44,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2d949d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:44,028 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:15:44,028 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:15:44,029 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:15:44,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:15:44,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:15:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18bc44e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:15:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:15:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:44,031 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:15:44,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76989242, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:44,033 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@341761a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:44,033 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:15:44,033 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:15:44,034 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:15:44,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:15:44,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:15:44,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fefb3b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:44,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:15:44,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:15:44,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:44,037 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:15:44,037 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@429fa85b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:44,038 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27574102, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:44,038 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:15:44,038 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:15:44,039 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:15:44,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:15:44,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:15:44,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4546dc0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:44,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:15:44,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:15:44,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:15:44,041 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:15:44,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d82861c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:15:44,042 DEBUG [Thread-1166 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:15:44,042 DEBUG [Thread-1175 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:15:44,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:15:44,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:15:44,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:15:44,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:15:44,044 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:15:44,044 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:15:44,045 DEBUG [Thread-1177 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:15:44,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:15:44,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:15:44,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:15:44,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:15:44,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:15:44,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:15:44,053 DEBUG [Thread-1179 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:15:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:15:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:15:44,055 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:15:44,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:15:44,059 DEBUG [Thread-1162 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:15:44,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:15:44,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:15:44,065 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:15:44,066 DEBUG [Thread-1173 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:15:44,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:15:44,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:15:44,068 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:15:44,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:15:44,069 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:15:44,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:15:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-12-10T07:15:44,072 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:15:44,073 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:15:44,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:15:44,074 DEBUG [Thread-1171 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:15:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:44,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T07:15:44,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:15:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-10T07:15:44,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:44,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:15:44,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:44,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:15:44,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:15:44,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:44,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:15:44,080 DEBUG [Thread-1160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:15:44,081 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:15:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:15:44,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:15:44,083 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:15:44,084 DEBUG [Thread-1168 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:15:44,085 DEBUG [Thread-1164 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:15:44,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:15:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:15:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:15:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:15:44,089 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:15:44,090 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:15:44,090 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:15:44,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:15:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:15:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:15:44,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/3c33a1519f0a41dba7159a0ea163a602 is 50, key is test_row_0/A:col10/1733814944075/Put/seqid=0 2024-12-10T07:15:44,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742075_1251 (size=14341) 2024-12-10T07:15:44,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733814944120, max timestamp=1733814944129Num uniques 859; 2024-12-10T07:15:44,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T07:15:44,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733814944120, max timestamp=1733814944131Num uniques 859; 2024-12-10T07:15:44,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T07:15:44,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/3c33a1519f0a41dba7159a0ea163a602 2024-12-10T07:15:44,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733814944120, max timestamp=1733814944131Num uniques 859; 2024-12-10T07:15:44,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T07:15:44,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/7b5057f16f164f42b5cf9710c670b9da is 50, key is test_row_0/B:col10/1733814944075/Put/seqid=0 2024-12-10T07:15:44,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:44,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815004164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:44,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:44,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815004166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:44,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:44,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:44,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:44,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815004167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:44,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:44,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:44,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815004173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:44,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:44,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815004173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:44,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:44,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-10T07:15:44,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T07:15:44,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:44,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:44,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:44,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:44,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:44,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:44,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742076_1252 (size=12001) 2024-12-10T07:15:44,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:44,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815004287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:44,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:44,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815004288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:44,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:44,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815004288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:44,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:44,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815004288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:44,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815004288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:44,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:44,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T07:15:44,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:44,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:44,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:44,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:44,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-10T07:15:44,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:44,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815004509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:44,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:44,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815004509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:44,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:44,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815004509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:44,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815004509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:44,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815004509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:44,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:44,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T07:15:44,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:44,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:44,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:44,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:44,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:44,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:44,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/7b5057f16f164f42b5cf9710c670b9da 2024-12-10T07:15:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-10T07:15:44,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/8ba9353b1f714e3c8f2c1190eb35b13d is 50, key is test_row_0/C:col10/1733814944075/Put/seqid=0 2024-12-10T07:15:44,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T07:15:44,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:44,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:44,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:44,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:44,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:44,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:44,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742077_1253 (size=12001) 2024-12-10T07:15:44,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/8ba9353b1f714e3c8f2c1190eb35b13d 2024-12-10T07:15:44,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/3c33a1519f0a41dba7159a0ea163a602 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/3c33a1519f0a41dba7159a0ea163a602 2024-12-10T07:15:44,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/3c33a1519f0a41dba7159a0ea163a602, entries=200, sequenceid=12, filesize=14.0 K 2024-12-10T07:15:44,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/7b5057f16f164f42b5cf9710c670b9da as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/7b5057f16f164f42b5cf9710c670b9da 2024-12-10T07:15:44,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/7b5057f16f164f42b5cf9710c670b9da, entries=150, sequenceid=12, filesize=11.7 K 2024-12-10T07:15:44,746 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-10T07:15:44,746 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-10T07:15:44,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/8ba9353b1f714e3c8f2c1190eb35b13d as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/8ba9353b1f714e3c8f2c1190eb35b13d 2024-12-10T07:15:44,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/8ba9353b1f714e3c8f2c1190eb35b13d, entries=150, sequenceid=12, filesize=11.7 K 2024-12-10T07:15:44,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 07d0f6bb02ffe54509ba6124b6830098 in 682ms, sequenceid=12, compaction requested=false 2024-12-10T07:15:44,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:44,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:44,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733814944159, max timestamp=1733814944828Num uniques 1018; 2024-12-10T07:15:44,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733814944159, max timestamp=1733814944828Num uniques 1018; 2024-12-10T07:15:44,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T07:15:44,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T07:15:44,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733814944159, max timestamp=1733814944828Num uniques 1018; 2024-12-10T07:15:44,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-10T07:15:44,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=40.25 KB heapSize=105.23 KB 2024-12-10T07:15:44,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:15:44,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:44,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:15:44,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:44,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:15:44,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:44,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/3bfb20db59b442b9a1ff5f9597e84f56 is 50, key is test_row_0/A:col10/1733814944833/Put/seqid=0 2024-12-10T07:15:44,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742078_1254 (size=14341) 2024-12-10T07:15:44,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/3bfb20db59b442b9a1ff5f9597e84f56 2024-12-10T07:15:44,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/e4220420895c4d6887c8992b7087d820 is 50, key is test_row_0/B:col10/1733814944833/Put/seqid=0 2024-12-10T07:15:44,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T07:15:44,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:44,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:44,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:44,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:44,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:44,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:44,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742079_1255 (size=12001) 2024-12-10T07:15:44,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814944862, max timestamp=1733814944886Num uniques 859; 2024-12-10T07:15:44,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:15:44,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814944862, max timestamp=1733814944886Num uniques 859; 2024-12-10T07:15:44,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:15:44,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814944862, max timestamp=1733814944886Num uniques 859; 2024-12-10T07:15:44,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:15:44,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/e4220420895c4d6887c8992b7087d820 2024-12-10T07:15:44,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/11432f19ddaa46219b595d6e9a0ecaff is 50, key is test_row_0/C:col10/1733814944833/Put/seqid=0 2024-12-10T07:15:44,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733814944917, max timestamp=1733814944942Num uniques 1068; 2024-12-10T07:15:44,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112  suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-10T07:15:44,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733814944917, max timestamp=1733814944942Num uniques 1068; 2024-12-10T07:15:44,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712  suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-10T07:15:44,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733814944917, max timestamp=1733814944942Num uniques 1068; 2024-12-10T07:15:44,952 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112  suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-10T07:15:44,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742080_1256 (size=12001) 2024-12-10T07:15:45,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T07:15:45,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:45,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:45,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:45,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:45,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:45,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:45,157 INFO [AsyncFSWAL-0-hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8-prefix:b1396b8d2d12,35871,1733814875835 {}] wal.AbstractFSWAL(1368): Slow sync cost: 166 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40613,DS-71c3e2fe-e06c-4f2c-995a-a2f36a2ad6fa,DISK]] 2024-12-10T07:15:45,157 INFO [AsyncFSWAL-0-hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8-prefix:b1396b8d2d12,35871,1733814875835 {}] wal.AbstractFSWAL(1368): Slow sync cost: 165 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40613,DS-71c3e2fe-e06c-4f2c-995a-a2f36a2ad6fa,DISK]] 2024-12-10T07:15:45,157 INFO [AsyncFSWAL-0-hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8-prefix:b1396b8d2d12,35871,1733814875835 {}] wal.AbstractFSWAL(1368): Slow sync cost: 154 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40613,DS-71c3e2fe-e06c-4f2c-995a-a2f36a2ad6fa,DISK]] 2024-12-10T07:15:45,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:45,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:45,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815004980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:45,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815004982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:45,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:45,161 INFO [AsyncFSWAL-0-hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData-prefix:b1396b8d2d12,36259,1733814874775 {}] wal.AbstractFSWAL(1368): Slow sync cost: 133 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40613,DS-71c3e2fe-e06c-4f2c-995a-a2f36a2ad6fa,DISK]] 2024-12-10T07:15:45,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:45,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815005159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:45,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:45,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815005159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:45,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:45,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815005161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:45,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:45,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:45,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-10T07:15:45,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:45,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815005264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:45,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:45,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815005275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:45,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815005275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:45,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:45,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815005275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:45,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:45,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815005275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:45,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T07:15:45,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:45,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:45,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:45,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:45,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:45,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:45,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/11432f19ddaa46219b595d6e9a0ecaff 2024-12-10T07:15:45,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/3bfb20db59b442b9a1ff5f9597e84f56 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/3bfb20db59b442b9a1ff5f9597e84f56 2024-12-10T07:15:45,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/3bfb20db59b442b9a1ff5f9597e84f56, entries=200, sequenceid=53, filesize=14.0 K 2024-12-10T07:15:45,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/e4220420895c4d6887c8992b7087d820 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/e4220420895c4d6887c8992b7087d820 2024-12-10T07:15:45,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/e4220420895c4d6887c8992b7087d820, entries=150, sequenceid=53, filesize=11.7 K 2024-12-10T07:15:45,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/11432f19ddaa46219b595d6e9a0ecaff as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/11432f19ddaa46219b595d6e9a0ecaff 2024-12-10T07:15:45,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/11432f19ddaa46219b595d6e9a0ecaff, entries=150, sequenceid=53, filesize=11.7 K 2024-12-10T07:15:45,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~125.11 KB/128112, currentSize=143.13 KB/146560 for 07d0f6bb02ffe54509ba6124b6830098 in 593ms, sequenceid=53, compaction requested=false 2024-12-10T07:15:45,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:45,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-10T07:15:45,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:45,463 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=143.13 KB heapSize=404.45 KB 2024-12-10T07:15:45,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:15:45,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:45,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:15:45,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:45,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:15:45,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:45,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/1af07eb810e7405d8271a04e32fc8305 is 50, key is test_row_0/A:col10/1733814944979/Put/seqid=0 2024-12-10T07:15:45,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:45,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742081_1257 (size=12001) 2024-12-10T07:15:45,500 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/1af07eb810e7405d8271a04e32fc8305 2024-12-10T07:15:45,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815005495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:45,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815005497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:45,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:45,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:45,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:45,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815005502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:45,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:45,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:45,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815005505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:45,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:45,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815005503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:45,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:45,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:45,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/35128a9493474764ad55075063713ef1 is 50, key is test_row_0/B:col10/1733814944979/Put/seqid=0 2024-12-10T07:15:45,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742082_1258 (size=12001) 2024-12-10T07:15:45,542 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/35128a9493474764ad55075063713ef1 2024-12-10T07:15:45,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/19f18c53be5a44b4b4633190f1c05cab is 50, key is test_row_0/C:col10/1733814944979/Put/seqid=0 2024-12-10T07:15:45,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742083_1259 (size=12001) 2024-12-10T07:15:45,605 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/19f18c53be5a44b4b4633190f1c05cab 2024-12-10T07:15:45,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/1af07eb810e7405d8271a04e32fc8305 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/1af07eb810e7405d8271a04e32fc8305 2024-12-10T07:15:45,618 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/1af07eb810e7405d8271a04e32fc8305, entries=150, sequenceid=107, filesize=11.7 K 2024-12-10T07:15:45,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815005617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:45,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815005617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815005618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:45,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815005618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:45,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/35128a9493474764ad55075063713ef1 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/35128a9493474764ad55075063713ef1 2024-12-10T07:15:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:45,627 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/35128a9493474764ad55075063713ef1, entries=150, sequenceid=107, filesize=11.7 K 2024-12-10T07:15:45,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/19f18c53be5a44b4b4633190f1c05cab as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/19f18c53be5a44b4b4633190f1c05cab 2024-12-10T07:15:45,635 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/19f18c53be5a44b4b4633190f1c05cab, entries=150, sequenceid=107, filesize=11.7 K 2024-12-10T07:15:45,637 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~143.13 KB/146560, heapSize ~404.41 KB/414112, currentSize=53.67 KB/54960 for 07d0f6bb02ffe54509ba6124b6830098 in 174ms, sequenceid=107, compaction requested=true 2024-12-10T07:15:45,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:45,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:45,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-12-10T07:15:45,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-12-10T07:15:45,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T07:15:45,640 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-10T07:15:45,640 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5650 sec 2024-12-10T07:15:45,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:15:45,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:45,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:15:45,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:45,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:15:45,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:45,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:45,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.5720 sec 2024-12-10T07:15:45,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/a14cbf2cb90e43a88b1830532dafd799 is 50, key is test_row_0/A:col10/1733814945499/Put/seqid=0 2024-12-10T07:15:45,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742084_1260 (size=12001) 2024-12-10T07:15:45,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/a14cbf2cb90e43a88b1830532dafd799 2024-12-10T07:15:45,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/782c8f98a8df43248dc2fe922a04bb49 is 50, key is test_row_0/B:col10/1733814945499/Put/seqid=0 2024-12-10T07:15:45,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742085_1261 (size=12001) 2024-12-10T07:15:45,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:45,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815005782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:45,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:45,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:45,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815005825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:45,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:45,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:45,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815005825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:45,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:45,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:45,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815005825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:45,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:45,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:45,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815005834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:45,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:45,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815005895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:45,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:45,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:46,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815006105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:46,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:46,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:46,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:46,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815006135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:46,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:46,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:46,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:46,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:46,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815006136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:46,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815006144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:46,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:46,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:46,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:46,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:46,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815006145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:46,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:46,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:46,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:46,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/782c8f98a8df43248dc2fe922a04bb49 2024-12-10T07:15:46,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/605476f6eaf74f0793166c946f4cb466 is 50, key is test_row_0/C:col10/1733814945499/Put/seqid=0 2024-12-10T07:15:46,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742086_1262 (size=12001) 2024-12-10T07:15:46,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/605476f6eaf74f0793166c946f4cb466 2024-12-10T07:15:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-10T07:15:46,228 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:15:46,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/a14cbf2cb90e43a88b1830532dafd799 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/a14cbf2cb90e43a88b1830532dafd799 2024-12-10T07:15:46,233 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:15:46,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-12-10T07:15:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-10T07:15:46,235 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:15:46,239 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:15:46,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:15:46,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/a14cbf2cb90e43a88b1830532dafd799, entries=150, sequenceid=119, filesize=11.7 K 2024-12-10T07:15:46,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/782c8f98a8df43248dc2fe922a04bb49 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/782c8f98a8df43248dc2fe922a04bb49 2024-12-10T07:15:46,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/782c8f98a8df43248dc2fe922a04bb49, entries=150, sequenceid=119, filesize=11.7 K 2024-12-10T07:15:46,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/605476f6eaf74f0793166c946f4cb466 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/605476f6eaf74f0793166c946f4cb466 2024-12-10T07:15:46,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/605476f6eaf74f0793166c946f4cb466, entries=150, sequenceid=119, filesize=11.7 K 2024-12-10T07:15:46,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 07d0f6bb02ffe54509ba6124b6830098 in 652ms, sequenceid=119, compaction requested=true 2024-12-10T07:15:46,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:46,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:15:46,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:46,292 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:15:46,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:15:46,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:46,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:15:46,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T07:15:46,292 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:15:46,295 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:15:46,295 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:15:46,295 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/A is initiating minor compaction (all files) 2024-12-10T07:15:46,295 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/B is initiating minor compaction (all files) 2024-12-10T07:15:46,295 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/B in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:46,295 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/A in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:46,295 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/7b5057f16f164f42b5cf9710c670b9da, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/e4220420895c4d6887c8992b7087d820, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/35128a9493474764ad55075063713ef1, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/782c8f98a8df43248dc2fe922a04bb49] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=46.9 K 2024-12-10T07:15:46,295 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/3c33a1519f0a41dba7159a0ea163a602, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/3bfb20db59b442b9a1ff5f9597e84f56, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/1af07eb810e7405d8271a04e32fc8305, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/a14cbf2cb90e43a88b1830532dafd799] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=51.4 K 2024-12-10T07:15:46,295 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b5057f16f164f42b5cf9710c670b9da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733814944069 2024-12-10T07:15:46,295 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c33a1519f0a41dba7159a0ea163a602, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733814944058 2024-12-10T07:15:46,296 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting e4220420895c4d6887c8992b7087d820, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733814944167 2024-12-10T07:15:46,296 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bfb20db59b442b9a1ff5f9597e84f56, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733814944167 2024-12-10T07:15:46,296 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 35128a9493474764ad55075063713ef1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733814944979 2024-12-10T07:15:46,296 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1af07eb810e7405d8271a04e32fc8305, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733814944979 2024-12-10T07:15:46,296 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting a14cbf2cb90e43a88b1830532dafd799, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733814945490 2024-12-10T07:15:46,296 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 782c8f98a8df43248dc2fe922a04bb49, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733814945490 2024-12-10T07:15:46,317 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#B#compaction#212 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:46,318 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/61887389fd204d409311cf3d4f4ffeb4 is 50, key is test_row_0/B:col10/1733814945499/Put/seqid=0 2024-12-10T07:15:46,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-10T07:15:46,347 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#A#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:46,348 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/a575bd6dd3004618a7c353053158c424 is 50, key is test_row_0/A:col10/1733814945499/Put/seqid=0 2024-12-10T07:15:46,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742087_1263 (size=12139) 2024-12-10T07:15:46,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742088_1264 (size=12139) 2024-12-10T07:15:46,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-10T07:15:46,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:46,401 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T07:15:46,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:15:46,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:46,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:15:46,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:46,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:15:46,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:46,410 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/a575bd6dd3004618a7c353053158c424 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/a575bd6dd3004618a7c353053158c424 2024-12-10T07:15:46,412 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/61887389fd204d409311cf3d4f4ffeb4 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/61887389fd204d409311cf3d4f4ffeb4 2024-12-10T07:15:46,425 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/B of 07d0f6bb02ffe54509ba6124b6830098 into 61887389fd204d409311cf3d4f4ffeb4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:46,426 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:46,426 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/B, priority=12, startTime=1733814946292; duration=0sec 2024-12-10T07:15:46,426 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:46,426 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:B 2024-12-10T07:15:46,426 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:15:46,427 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:15:46,428 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/C is initiating minor compaction (all files) 2024-12-10T07:15:46,428 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/C in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:46,428 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/8ba9353b1f714e3c8f2c1190eb35b13d, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/11432f19ddaa46219b595d6e9a0ecaff, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/19f18c53be5a44b4b4633190f1c05cab, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/605476f6eaf74f0793166c946f4cb466] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=46.9 K 2024-12-10T07:15:46,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/315e396bdfe4468bae7161b9acab02b4 is 50, key is test_row_0/A:col10/1733814945771/Put/seqid=0 2024-12-10T07:15:46,429 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ba9353b1f714e3c8f2c1190eb35b13d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733814944069 2024-12-10T07:15:46,430 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 11432f19ddaa46219b595d6e9a0ecaff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733814944167 2024-12-10T07:15:46,433 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 19f18c53be5a44b4b4633190f1c05cab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733814944979 2024-12-10T07:15:46,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:46,436 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 605476f6eaf74f0793166c946f4cb466, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733814945490 2024-12-10T07:15:46,437 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/A of 07d0f6bb02ffe54509ba6124b6830098 into a575bd6dd3004618a7c353053158c424(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:46,437 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:46,437 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/A, priority=12, startTime=1733814946292; duration=0sec 2024-12-10T07:15:46,437 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:46,437 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:A 2024-12-10T07:15:46,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742089_1265 (size=12151) 2024-12-10T07:15:46,451 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#C#compaction#215 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:46,452 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/e7620151c98f4ae4ad80b842a02280f7 is 50, key is test_row_0/C:col10/1733814945499/Put/seqid=0 2024-12-10T07:15:46,456 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/315e396bdfe4468bae7161b9acab02b4 2024-12-10T07:15:46,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742090_1266 (size=12139) 2024-12-10T07:15:46,509 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/e7620151c98f4ae4ad80b842a02280f7 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/e7620151c98f4ae4ad80b842a02280f7 2024-12-10T07:15:46,519 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/C of 07d0f6bb02ffe54509ba6124b6830098 into e7620151c98f4ae4ad80b842a02280f7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:46,519 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:46,519 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/C, priority=12, startTime=1733814946292; duration=0sec 2024-12-10T07:15:46,519 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:46,520 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:C 2024-12-10T07:15:46,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/00b26a55a948407ab95f4ce493bf2c02 is 50, key is test_row_0/B:col10/1733814945771/Put/seqid=0 2024-12-10T07:15:46,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-10T07:15:46,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815006561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:46,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:46,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:46,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:46,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742091_1267 (size=12151) 2024-12-10T07:15:46,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:46,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815006647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:46,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:46,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:46,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:46,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815006657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:46,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:46,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:46,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:46,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815006657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:46,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:46,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:46,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:46,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815006660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:46,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:46,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:46,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:46,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:46,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815006675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:46,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:46,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:46,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:46,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-10T07:15:46,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:46,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815006889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:46,983 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/00b26a55a948407ab95f4ce493bf2c02 2024-12-10T07:15:47,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/2f5e29906ad8431a8722f04e2423438c is 50, key is test_row_0/C:col10/1733814945771/Put/seqid=0 2024-12-10T07:15:47,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742092_1268 (size=12151) 2024-12-10T07:15:47,067 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/2f5e29906ad8431a8722f04e2423438c 2024-12-10T07:15:47,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/315e396bdfe4468bae7161b9acab02b4 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/315e396bdfe4468bae7161b9acab02b4 2024-12-10T07:15:47,085 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/315e396bdfe4468bae7161b9acab02b4, entries=150, sequenceid=139, filesize=11.9 K 2024-12-10T07:15:47,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/00b26a55a948407ab95f4ce493bf2c02 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/00b26a55a948407ab95f4ce493bf2c02 2024-12-10T07:15:47,099 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/00b26a55a948407ab95f4ce493bf2c02, entries=150, sequenceid=139, filesize=11.9 K 2024-12-10T07:15:47,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/2f5e29906ad8431a8722f04e2423438c as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/2f5e29906ad8431a8722f04e2423438c 2024-12-10T07:15:47,112 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/2f5e29906ad8431a8722f04e2423438c, entries=150, sequenceid=139, filesize=11.9 K 2024-12-10T07:15:47,116 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 07d0f6bb02ffe54509ba6124b6830098 in 715ms, sequenceid=139, compaction requested=false 2024-12-10T07:15:47,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:47,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:47,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-12-10T07:15:47,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-12-10T07:15:47,132 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-10T07:15:47,132 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 890 msec 2024-12-10T07:15:47,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 901 msec 2024-12-10T07:15:47,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T07:15:47,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:15:47,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:47,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:15:47,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:47,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:15:47,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:47,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/9096ae1774af4231a5ec0501e407f77e is 50, key is test_row_0/A:col10/1733814946509/Put/seqid=0 2024-12-10T07:15:47,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742093_1269 (size=14541) 2024-12-10T07:15:47,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/9096ae1774af4231a5ec0501e407f77e 2024-12-10T07:15:47,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/ece38bfb6fbe490ba9e192e9f3479719 is 50, key is test_row_0/B:col10/1733814946509/Put/seqid=0 2024-12-10T07:15:47,315 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T07:15:47,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:47,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815007354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:47,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742094_1270 (size=12151) 2024-12-10T07:15:47,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/ece38bfb6fbe490ba9e192e9f3479719 2024-12-10T07:15:47,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-10T07:15:47,375 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:15:47,376 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:15:47,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-10T07:15:47,379 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:15:47,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-10T07:15:47,380 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:15:47,380 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:15:47,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/b2ac7f71811a43999ec692186536f8b8 is 50, key is test_row_0/C:col10/1733814946509/Put/seqid=0 2024-12-10T07:15:47,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742095_1271 (size=12151) 2024-12-10T07:15:47,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/b2ac7f71811a43999ec692186536f8b8 2024-12-10T07:15:47,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/9096ae1774af4231a5ec0501e407f77e as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/9096ae1774af4231a5ec0501e407f77e 2024-12-10T07:15:47,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/9096ae1774af4231a5ec0501e407f77e, entries=200, sequenceid=155, filesize=14.2 K 2024-12-10T07:15:47,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/ece38bfb6fbe490ba9e192e9f3479719 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/ece38bfb6fbe490ba9e192e9f3479719 2024-12-10T07:15:47,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815007465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:47,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:47,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:47,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:47,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/ece38bfb6fbe490ba9e192e9f3479719, entries=150, sequenceid=155, filesize=11.9 K 2024-12-10T07:15:47,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/b2ac7f71811a43999ec692186536f8b8 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/b2ac7f71811a43999ec692186536f8b8 2024-12-10T07:15:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-10T07:15:47,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/b2ac7f71811a43999ec692186536f8b8, entries=150, sequenceid=155, filesize=11.9 K 2024-12-10T07:15:47,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 07d0f6bb02ffe54509ba6124b6830098 in 280ms, sequenceid=155, compaction requested=true 2024-12-10T07:15:47,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:47,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:15:47,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:47,492 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:47,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:15:47,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:47,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:15:47,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T07:15:47,493 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:47,494 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38831 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:47,494 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/A is initiating minor compaction (all files) 2024-12-10T07:15:47,494 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/A in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:47,494 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/a575bd6dd3004618a7c353053158c424, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/315e396bdfe4468bae7161b9acab02b4, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/9096ae1774af4231a5ec0501e407f77e] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=37.9 K 2024-12-10T07:15:47,495 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36441 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:47,495 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/B is initiating minor compaction (all files) 2024-12-10T07:15:47,495 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/B in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:47,495 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/61887389fd204d409311cf3d4f4ffeb4, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/00b26a55a948407ab95f4ce493bf2c02, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/ece38bfb6fbe490ba9e192e9f3479719] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=35.6 K 2024-12-10T07:15:47,496 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 61887389fd204d409311cf3d4f4ffeb4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733814945490 2024-12-10T07:15:47,496 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting a575bd6dd3004618a7c353053158c424, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733814945490 2024-12-10T07:15:47,497 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 00b26a55a948407ab95f4ce493bf2c02, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733814945748 2024-12-10T07:15:47,497 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 315e396bdfe4468bae7161b9acab02b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733814945748 2024-12-10T07:15:47,497 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting ece38bfb6fbe490ba9e192e9f3479719, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733814946509 2024-12-10T07:15:47,498 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9096ae1774af4231a5ec0501e407f77e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733814946509 2024-12-10T07:15:47,514 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#B#compaction#221 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:47,514 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/fd3c075af7dc484d87652c1dad982f68 is 50, key is test_row_0/B:col10/1733814946509/Put/seqid=0 2024-12-10T07:15:47,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-10T07:15:47,536 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#A#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:47,537 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/9b62d060936c4bcab9a33123f017ef74 is 50, key is test_row_0/A:col10/1733814946509/Put/seqid=0 2024-12-10T07:15:47,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:47,540 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T07:15:47,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:15:47,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:47,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:15:47,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:47,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:15:47,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:47,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/dbcf974c100d4946af77d8a136193327 is 50, key is test_row_0/A:col10/1733814947306/Put/seqid=0 2024-12-10T07:15:47,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742096_1272 (size=12391) 2024-12-10T07:15:47,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742097_1273 (size=12391) 2024-12-10T07:15:47,620 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/fd3c075af7dc484d87652c1dad982f68 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/fd3c075af7dc484d87652c1dad982f68 2024-12-10T07:15:47,630 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/9b62d060936c4bcab9a33123f017ef74 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/9b62d060936c4bcab9a33123f017ef74 2024-12-10T07:15:47,655 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/B of 07d0f6bb02ffe54509ba6124b6830098 into fd3c075af7dc484d87652c1dad982f68(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:47,655 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:47,655 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/B, priority=13, startTime=1733814947492; duration=0sec 2024-12-10T07:15:47,655 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:47,655 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:B 2024-12-10T07:15:47,655 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:47,657 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/A of 07d0f6bb02ffe54509ba6124b6830098 into 9b62d060936c4bcab9a33123f017ef74(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:47,657 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:47,657 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/A, priority=13, startTime=1733814947492; duration=0sec 2024-12-10T07:15:47,657 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:47,657 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:A 2024-12-10T07:15:47,658 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36441 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:47,658 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/C is initiating minor compaction (all files) 2024-12-10T07:15:47,658 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/C in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:47,658 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/e7620151c98f4ae4ad80b842a02280f7, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/2f5e29906ad8431a8722f04e2423438c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/b2ac7f71811a43999ec692186536f8b8] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=35.6 K 2024-12-10T07:15:47,659 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting e7620151c98f4ae4ad80b842a02280f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733814945490 2024-12-10T07:15:47,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742098_1274 (size=12151) 2024-12-10T07:15:47,665 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f5e29906ad8431a8722f04e2423438c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733814945748 2024-12-10T07:15:47,666 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting b2ac7f71811a43999ec692186536f8b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733814946509 2024-12-10T07:15:47,673 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/dbcf974c100d4946af77d8a136193327 2024-12-10T07:15:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:47,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:47,691 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#C#compaction#224 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:47,691 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/4e866abe61aa4c5ab89e827a657882e5 is 50, key is test_row_0/C:col10/1733814946509/Put/seqid=0 2024-12-10T07:15:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-10T07:15:47,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/19e4634d0c514ee38e84887b070c6518 is 50, key is test_row_0/B:col10/1733814947306/Put/seqid=0 2024-12-10T07:15:47,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815007699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:47,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:47,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:47,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:47,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815007703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:47,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:47,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:47,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:47,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815007703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:47,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815007708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:47,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815007711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:47,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:47,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:47,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:47,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742099_1275 (size=12391) 2024-12-10T07:15:47,761 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/4e866abe61aa4c5ab89e827a657882e5 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/4e866abe61aa4c5ab89e827a657882e5 2024-12-10T07:15:47,770 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/C of 07d0f6bb02ffe54509ba6124b6830098 into 4e866abe61aa4c5ab89e827a657882e5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:47,770 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:47,770 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/C, priority=13, startTime=1733814947492; duration=0sec 2024-12-10T07:15:47,770 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:47,770 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:C 2024-12-10T07:15:47,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742100_1276 (size=12151) 2024-12-10T07:15:47,779 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/19e4634d0c514ee38e84887b070c6518 2024-12-10T07:15:47,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/1792ab05143d4ca2b73a9c7764ff63bd is 50, key is test_row_0/C:col10/1733814947306/Put/seqid=0 2024-12-10T07:15:47,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815007819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:47,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:47,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815007825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:47,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:47,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:47,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:47,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815007825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:47,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815007825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:47,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815007825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:47,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:47,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:47,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:47,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:47,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742101_1277 (size=12151) 2024-12-10T07:15:48,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-10T07:15:48,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:48,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:48,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815008068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:48,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815008067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:48,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:48,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815008068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:48,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:48,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815008068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:48,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:48,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:48,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:48,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815008067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:48,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:48,239 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/1792ab05143d4ca2b73a9c7764ff63bd 2024-12-10T07:15:48,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/dbcf974c100d4946af77d8a136193327 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/dbcf974c100d4946af77d8a136193327 2024-12-10T07:15:48,256 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/dbcf974c100d4946af77d8a136193327, entries=150, sequenceid=174, filesize=11.9 K 2024-12-10T07:15:48,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/19e4634d0c514ee38e84887b070c6518 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/19e4634d0c514ee38e84887b070c6518 2024-12-10T07:15:48,284 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/19e4634d0c514ee38e84887b070c6518, entries=150, sequenceid=174, filesize=11.9 K 2024-12-10T07:15:48,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/1792ab05143d4ca2b73a9c7764ff63bd as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/1792ab05143d4ca2b73a9c7764ff63bd 2024-12-10T07:15:48,306 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/1792ab05143d4ca2b73a9c7764ff63bd, entries=150, sequenceid=174, filesize=11.9 K 2024-12-10T07:15:48,307 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 07d0f6bb02ffe54509ba6124b6830098 in 767ms, sequenceid=174, compaction requested=false 2024-12-10T07:15:48,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:48,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:48,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-10T07:15:48,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-10T07:15:48,310 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-10T07:15:48,310 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 928 msec 2024-12-10T07:15:48,312 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 935 msec 2024-12-10T07:15:48,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T07:15:48,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:15:48,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:48,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:15:48,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:48,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:15:48,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:48,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:48,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/5626c716617b40f08f830ff561777947 is 50, key is test_row_0/A:col10/1733814947697/Put/seqid=0 2024-12-10T07:15:48,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742102_1278 (size=14541) 2024-12-10T07:15:48,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/5626c716617b40f08f830ff561777947 2024-12-10T07:15:48,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/7b0ccb475674452b8c8376a9590965f7 is 50, key is test_row_0/B:col10/1733814947697/Put/seqid=0 2024-12-10T07:15:48,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742103_1279 (size=12151) 2024-12-10T07:15:48,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:48,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815008454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:48,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:48,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815008454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:48,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:48,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815008463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:48,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815008464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:48,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815008459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:48,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:48,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-10T07:15:48,526 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:15:48,527 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:15:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-10T07:15:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T07:15:48,531 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:15:48,532 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:15:48,532 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:15:48,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815008575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:48,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815008575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:48,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815008575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:48,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:48,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815008575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:48,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815008576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T07:15:48,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-10T07:15:48,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:48,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:48,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:48,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:48,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:48,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:48,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:48,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815008786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815008786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:48,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815008786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:48,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815008786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:48,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:48,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:48,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:48,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815008791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:48,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:48,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:48,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T07:15:48,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-10T07:15:48,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:48,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:48,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:48,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:48,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:48,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/7b0ccb475674452b8c8376a9590965f7 2024-12-10T07:15:48,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:48,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/7b2708d1138e434bada57861edcd9dbd is 50, key is test_row_0/C:col10/1733814947697/Put/seqid=0 2024-12-10T07:15:48,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742104_1280 (size=12151) 2024-12-10T07:15:48,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/7b2708d1138e434bada57861edcd9dbd 2024-12-10T07:15:48,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/5626c716617b40f08f830ff561777947 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/5626c716617b40f08f830ff561777947 2024-12-10T07:15:48,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/5626c716617b40f08f830ff561777947, entries=200, sequenceid=191, filesize=14.2 K 2024-12-10T07:15:48,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/7b0ccb475674452b8c8376a9590965f7 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/7b0ccb475674452b8c8376a9590965f7 2024-12-10T07:15:48,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/7b0ccb475674452b8c8376a9590965f7, entries=150, sequenceid=191, filesize=11.9 K 2024-12-10T07:15:48,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/7b2708d1138e434bada57861edcd9dbd as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/7b2708d1138e434bada57861edcd9dbd 2024-12-10T07:15:48,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/7b2708d1138e434bada57861edcd9dbd, entries=150, sequenceid=191, filesize=11.9 K 2024-12-10T07:15:48,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 07d0f6bb02ffe54509ba6124b6830098 in 562ms, sequenceid=191, compaction requested=true 2024-12-10T07:15:48,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:48,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:15:48,944 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:48,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:48,944 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:48,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:15:48,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:48,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:15:48,944 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:48,945 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39083 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:48,945 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:48,945 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/B is initiating minor compaction (all files) 2024-12-10T07:15:48,945 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/A is initiating minor compaction (all files) 2024-12-10T07:15:48,945 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/A in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:48,945 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/B in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:48,945 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/fd3c075af7dc484d87652c1dad982f68, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/19e4634d0c514ee38e84887b070c6518, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/7b0ccb475674452b8c8376a9590965f7] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=35.8 K 2024-12-10T07:15:48,945 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/9b62d060936c4bcab9a33123f017ef74, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/dbcf974c100d4946af77d8a136193327, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/5626c716617b40f08f830ff561777947] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=38.2 K 2024-12-10T07:15:48,947 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting fd3c075af7dc484d87652c1dad982f68, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733814946509 2024-12-10T07:15:48,948 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b62d060936c4bcab9a33123f017ef74, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733814946509 2024-12-10T07:15:48,952 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 19e4634d0c514ee38e84887b070c6518, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733814947306 2024-12-10T07:15:48,953 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbcf974c100d4946af77d8a136193327, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733814947306 2024-12-10T07:15:48,954 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b0ccb475674452b8c8376a9590965f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733814947680 2024-12-10T07:15:48,956 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5626c716617b40f08f830ff561777947, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733814947680 2024-12-10T07:15:48,984 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#A#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:48,984 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/21199358a9654b21b93efdca05dfc728 is 50, key is test_row_0/A:col10/1733814947697/Put/seqid=0 2024-12-10T07:15:49,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742105_1281 (size=12493) 2024-12-10T07:15:49,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-10T07:15:49,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:49,009 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T07:15:49,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:15:49,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:49,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:15:49,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:49,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:15:49,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:49,015 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#B#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:49,017 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/04286781c03c4389b124ade95741279a is 50, key is test_row_0/B:col10/1733814947697/Put/seqid=0 2024-12-10T07:15:49,029 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/21199358a9654b21b93efdca05dfc728 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/21199358a9654b21b93efdca05dfc728 2024-12-10T07:15:49,035 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/A of 07d0f6bb02ffe54509ba6124b6830098 into 21199358a9654b21b93efdca05dfc728(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:49,036 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:49,036 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/A, priority=13, startTime=1733814948943; duration=0sec 2024-12-10T07:15:49,036 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:49,036 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:A 2024-12-10T07:15:49,036 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:49,038 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:49,038 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/C is initiating minor compaction (all files) 2024-12-10T07:15:49,039 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/C in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:49,039 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/4e866abe61aa4c5ab89e827a657882e5, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/1792ab05143d4ca2b73a9c7764ff63bd, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/7b2708d1138e434bada57861edcd9dbd] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=35.8 K 2024-12-10T07:15:49,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/606fa2a0888b41ee962205276a9dffb4 is 50, key is test_row_0/A:col10/1733814948420/Put/seqid=0 2024-12-10T07:15:49,043 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e866abe61aa4c5ab89e827a657882e5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733814946509 2024-12-10T07:15:49,043 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1792ab05143d4ca2b73a9c7764ff63bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733814947306 2024-12-10T07:15:49,043 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b2708d1138e434bada57861edcd9dbd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733814947680 2024-12-10T07:15:49,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742106_1282 (size=12493) 2024-12-10T07:15:49,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742107_1283 (size=12151) 2024-12-10T07:15:49,071 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/606fa2a0888b41ee962205276a9dffb4 2024-12-10T07:15:49,072 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#C#compaction#233 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:49,072 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/2f66fcd28dd24de9b9d5f13f8bb9551e is 50, key is test_row_0/C:col10/1733814947697/Put/seqid=0 2024-12-10T07:15:49,093 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/04286781c03c4389b124ade95741279a as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/04286781c03c4389b124ade95741279a 2024-12-10T07:15:49,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742108_1284 (size=12493) 2024-12-10T07:15:49,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:49,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:49,119 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/B of 07d0f6bb02ffe54509ba6124b6830098 into 04286781c03c4389b124ade95741279a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:49,119 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:49,119 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/B, priority=13, startTime=1733814948944; duration=0sec 2024-12-10T07:15:49,119 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:49,119 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:B 2024-12-10T07:15:49,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/3f0d1d13291d4f36aae87f77ee11b9f3 is 50, key is test_row_0/B:col10/1733814948420/Put/seqid=0 2024-12-10T07:15:49,125 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/2f66fcd28dd24de9b9d5f13f8bb9551e as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/2f66fcd28dd24de9b9d5f13f8bb9551e 2024-12-10T07:15:49,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:49,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815009119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:49,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:49,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:49,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815009122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:49,133 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/C of 07d0f6bb02ffe54509ba6124b6830098 into 2f66fcd28dd24de9b9d5f13f8bb9551e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:49,133 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:49,133 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/C, priority=13, startTime=1733814948944; duration=0sec 2024-12-10T07:15:49,133 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:49,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:49,133 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:C 2024-12-10T07:15:49,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815009125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:49,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:49,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815009125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:49,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:49,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815009127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:49,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:49,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742109_1285 (size=12151) 2024-12-10T07:15:49,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T07:15:49,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:49,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815009236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:49,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:49,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815009245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:49,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:49,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815009246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:49,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:49,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815009246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:49,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:49,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815009246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:49,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:49,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:49,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815009446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:49,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:49,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815009455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:49,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:49,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815009456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:49,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815009457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:49,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:49,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815009477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:49,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:49,562 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/3f0d1d13291d4f36aae87f77ee11b9f3 2024-12-10T07:15:49,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/b8d4c955bb074787a412243663dcfd70 is 50, key is test_row_0/C:col10/1733814948420/Put/seqid=0 2024-12-10T07:15:49,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742110_1286 (size=12151) 2024-12-10T07:15:49,592 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/b8d4c955bb074787a412243663dcfd70 2024-12-10T07:15:49,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/606fa2a0888b41ee962205276a9dffb4 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/606fa2a0888b41ee962205276a9dffb4 2024-12-10T07:15:49,623 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/606fa2a0888b41ee962205276a9dffb4, entries=150, sequenceid=209, filesize=11.9 K 2024-12-10T07:15:49,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/3f0d1d13291d4f36aae87f77ee11b9f3 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/3f0d1d13291d4f36aae87f77ee11b9f3 2024-12-10T07:15:49,643 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/3f0d1d13291d4f36aae87f77ee11b9f3, entries=150, sequenceid=209, filesize=11.9 K 2024-12-10T07:15:49,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/b8d4c955bb074787a412243663dcfd70 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/b8d4c955bb074787a412243663dcfd70 2024-12-10T07:15:49,648 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/b8d4c955bb074787a412243663dcfd70, entries=150, sequenceid=209, filesize=11.9 K 2024-12-10T07:15:49,651 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 07d0f6bb02ffe54509ba6124b6830098 in 642ms, sequenceid=209, compaction requested=false 2024-12-10T07:15:49,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:49,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:49,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-10T07:15:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-10T07:15:49,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-10T07:15:49,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1210 sec 2024-12-10T07:15:49,657 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.1290 sec 2024-12-10T07:15:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-10T07:15:49,665 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:15:49,666 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:15:49,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-10T07:15:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T07:15:49,680 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:15:49,687 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:15:49,687 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:15:49,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T07:15:49,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:15:49,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:49,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:15:49,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:49,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:15:49,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:49,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:49,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/436b21ca297a4955a5eb40061f4896ed is 50, key is test_row_0/A:col10/1733814949121/Put/seqid=0 2024-12-10T07:15:49,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742111_1287 (size=14541) 2024-12-10T07:15:49,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/436b21ca297a4955a5eb40061f4896ed 2024-12-10T07:15:49,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T07:15:49,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/eba764909bb64f1b975dd51a6fcd301c is 50, key is test_row_0/B:col10/1733814949121/Put/seqid=0 2024-12-10T07:15:49,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742112_1288 (size=12151) 2024-12-10T07:15:49,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:49,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815009799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:49,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:49,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:49,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815009801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:49,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:49,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:49,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815009805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:49,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815009807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:49,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815009806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:49,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-10T07:15:49,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:49,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:49,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:49,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:49,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:49,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:49,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815009915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:49,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815009915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815009915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815009915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:49,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815009917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:49,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:49,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T07:15:50,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-10T07:15:50,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:50,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:50,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:50,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:50,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:50,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:50,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815010126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:50,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815010126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815010126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:50,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815010126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:50,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:50,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815010135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:50,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/eba764909bb64f1b975dd51a6fcd301c 2024-12-10T07:15:50,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/05dbe89c336f45a0a43caec7abad853a is 50, key is test_row_0/C:col10/1733814949121/Put/seqid=0 2024-12-10T07:15:50,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-10T07:15:50,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:50,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:50,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:50,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:50,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:50,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:50,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742113_1289 (size=12151) 2024-12-10T07:15:50,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/05dbe89c336f45a0a43caec7abad853a 2024-12-10T07:15:50,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/436b21ca297a4955a5eb40061f4896ed as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/436b21ca297a4955a5eb40061f4896ed 2024-12-10T07:15:50,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/436b21ca297a4955a5eb40061f4896ed, entries=200, sequenceid=227, filesize=14.2 K 2024-12-10T07:15:50,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/eba764909bb64f1b975dd51a6fcd301c as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/eba764909bb64f1b975dd51a6fcd301c 2024-12-10T07:15:50,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/eba764909bb64f1b975dd51a6fcd301c, entries=150, sequenceid=227, filesize=11.9 K 2024-12-10T07:15:50,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/05dbe89c336f45a0a43caec7abad853a as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/05dbe89c336f45a0a43caec7abad853a 2024-12-10T07:15:50,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/05dbe89c336f45a0a43caec7abad853a, entries=150, sequenceid=227, filesize=11.9 K 2024-12-10T07:15:50,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 07d0f6bb02ffe54509ba6124b6830098 in 514ms, sequenceid=227, compaction requested=true 2024-12-10T07:15:50,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:50,273 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:50,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:15:50,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:50,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:15:50,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:50,273 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:50,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:15:50,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:50,273 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:50,274 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/A is initiating minor compaction (all files) 2024-12-10T07:15:50,274 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/A in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:50,274 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:50,274 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/B is initiating minor compaction (all files) 2024-12-10T07:15:50,274 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/21199358a9654b21b93efdca05dfc728, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/606fa2a0888b41ee962205276a9dffb4, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/436b21ca297a4955a5eb40061f4896ed] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=38.3 K 2024-12-10T07:15:50,274 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/B in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:50,274 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/04286781c03c4389b124ade95741279a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/3f0d1d13291d4f36aae87f77ee11b9f3, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/eba764909bb64f1b975dd51a6fcd301c] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=35.9 K 2024-12-10T07:15:50,274 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21199358a9654b21b93efdca05dfc728, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733814947680 2024-12-10T07:15:50,274 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 04286781c03c4389b124ade95741279a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733814947680 2024-12-10T07:15:50,275 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f0d1d13291d4f36aae87f77ee11b9f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733814948420 2024-12-10T07:15:50,275 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 606fa2a0888b41ee962205276a9dffb4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733814948420 2024-12-10T07:15:50,275 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting eba764909bb64f1b975dd51a6fcd301c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733814949121 2024-12-10T07:15:50,275 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 436b21ca297a4955a5eb40061f4896ed, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733814949112 2024-12-10T07:15:50,287 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#B#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:50,287 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#A#compaction#239 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:50,287 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/6a136468a02a47c092ab09cd5fe509ba is 50, key is test_row_0/B:col10/1733814949121/Put/seqid=0 2024-12-10T07:15:50,287 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/56610babc4eb4a2db6b9903de51d5a58 is 50, key is test_row_0/A:col10/1733814949121/Put/seqid=0 2024-12-10T07:15:50,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742114_1290 (size=12595) 2024-12-10T07:15:50,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742115_1291 (size=12595) 2024-12-10T07:15:50,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T07:15:50,312 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/6a136468a02a47c092ab09cd5fe509ba as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/6a136468a02a47c092ab09cd5fe509ba 2024-12-10T07:15:50,318 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/B of 07d0f6bb02ffe54509ba6124b6830098 into 6a136468a02a47c092ab09cd5fe509ba(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:50,318 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:50,318 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/B, priority=13, startTime=1733814950273; duration=0sec 2024-12-10T07:15:50,318 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:50,318 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:B 2024-12-10T07:15:50,318 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:50,320 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:50,320 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/C is initiating minor compaction (all files) 2024-12-10T07:15:50,320 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/C in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:50,321 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/2f66fcd28dd24de9b9d5f13f8bb9551e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/b8d4c955bb074787a412243663dcfd70, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/05dbe89c336f45a0a43caec7abad853a] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=35.9 K 2024-12-10T07:15:50,321 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f66fcd28dd24de9b9d5f13f8bb9551e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733814947680 2024-12-10T07:15:50,321 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting b8d4c955bb074787a412243663dcfd70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733814948420 2024-12-10T07:15:50,322 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 05dbe89c336f45a0a43caec7abad853a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733814949121 2024-12-10T07:15:50,337 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#C#compaction#241 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:50,340 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/73d4454464aa43778c107034cfb6a279 is 50, key is test_row_0/C:col10/1733814949121/Put/seqid=0 2024-12-10T07:15:50,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742116_1292 (size=12595) 2024-12-10T07:15:50,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-10T07:15:50,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:50,381 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T07:15:50,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:15:50,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:50,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:15:50,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:50,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:15:50,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:50,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/86f51c63f9cb4051af0f224b13784b0d is 50, key is test_row_0/A:col10/1733814949804/Put/seqid=0 2024-12-10T07:15:50,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:50,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:50,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742117_1293 (size=12151) 2024-12-10T07:15:50,450 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/86f51c63f9cb4051af0f224b13784b0d 2024-12-10T07:15:50,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/3c4b716a77e943c79e0269e44af76148 is 50, key is test_row_0/B:col10/1733814949804/Put/seqid=0 2024-12-10T07:15:50,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:50,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815010457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815010455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:50,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:50,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815010465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:50,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:50,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815010469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:50,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:50,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815010473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:50,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:50,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742118_1294 (size=12151) 2024-12-10T07:15:50,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:50,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815010575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:50,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:50,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:50,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815010575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:50,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:50,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815010577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:50,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:50,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815010585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:50,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:50,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815010585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:50,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:50,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:50,722 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/56610babc4eb4a2db6b9903de51d5a58 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/56610babc4eb4a2db6b9903de51d5a58 2024-12-10T07:15:50,762 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/A of 07d0f6bb02ffe54509ba6124b6830098 into 56610babc4eb4a2db6b9903de51d5a58(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:50,762 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:50,762 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/A, priority=13, startTime=1733814950273; duration=0sec 2024-12-10T07:15:50,762 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:50,763 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:A 2024-12-10T07:15:50,796 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/73d4454464aa43778c107034cfb6a279 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/73d4454464aa43778c107034cfb6a279 2024-12-10T07:15:50,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815010788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:50,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:50,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815010785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:50,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:50,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815010786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:50,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:50,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815010795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:50,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:50,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:50,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815010797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T07:15:50,848 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/C of 07d0f6bb02ffe54509ba6124b6830098 into 73d4454464aa43778c107034cfb6a279(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:50,848 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:50,848 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/C, priority=13, startTime=1733814950273; duration=0sec 2024-12-10T07:15:50,848 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:50,848 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:C 2024-12-10T07:15:50,885 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/3c4b716a77e943c79e0269e44af76148 2024-12-10T07:15:50,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/ed32063d07c64e6e997bb5f54bd200b1 is 50, key is test_row_0/C:col10/1733814949804/Put/seqid=0 2024-12-10T07:15:50,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742119_1295 (size=12151) 2024-12-10T07:15:51,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:51,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815011105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:51,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:51,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815011105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:51,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815011105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:51,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:51,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:51,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815011105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:51,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815011105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:51,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:51,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:51,369 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/ed32063d07c64e6e997bb5f54bd200b1 2024-12-10T07:15:51,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/86f51c63f9cb4051af0f224b13784b0d as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/86f51c63f9cb4051af0f224b13784b0d 2024-12-10T07:15:51,391 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/86f51c63f9cb4051af0f224b13784b0d, entries=150, sequenceid=246, filesize=11.9 K 2024-12-10T07:15:51,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/3c4b716a77e943c79e0269e44af76148 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/3c4b716a77e943c79e0269e44af76148 2024-12-10T07:15:51,401 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/3c4b716a77e943c79e0269e44af76148, entries=150, sequenceid=246, filesize=11.9 K 2024-12-10T07:15:51,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/ed32063d07c64e6e997bb5f54bd200b1 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/ed32063d07c64e6e997bb5f54bd200b1 2024-12-10T07:15:51,408 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/ed32063d07c64e6e997bb5f54bd200b1, entries=150, sequenceid=246, filesize=11.9 K 2024-12-10T07:15:51,409 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 07d0f6bb02ffe54509ba6124b6830098 in 1028ms, sequenceid=246, compaction requested=false 2024-12-10T07:15:51,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:51,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:51,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-10T07:15:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-10T07:15:51,414 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-10T07:15:51,414 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7240 sec 2024-12-10T07:15:51,416 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.7490 sec 2024-12-10T07:15:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:51,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T07:15:51,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:15:51,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:51,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:15:51,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:51,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:15:51,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:51,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/1b001dd018e24955b8ecbc7ea9eeacf0 is 50, key is test_row_0/A:col10/1733814951616/Put/seqid=0 2024-12-10T07:15:51,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815011630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:51,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:51,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815011632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:51,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:51,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815011639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:51,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815011641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:51,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815011642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:51,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:51,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:51,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742120_1296 (size=12301) 2024-12-10T07:15:51,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815011744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:51,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815011744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:51,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:51,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815011756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:51,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:51,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815011756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:51,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815011756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:51,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-10T07:15:51,825 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:15:51,829 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:15:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-10T07:15:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T07:15:51,849 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:15:51,853 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:15:51,853 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:15:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T07:15:51,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:51,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815011954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:51,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:51,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:51,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815011954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:51,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:51,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:51,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815011965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:51,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:51,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:51,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:51,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815011965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:51,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815011965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:51,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:51,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:51,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:52,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-10T07:15:52,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:52,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:52,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:52,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:52,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:52,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/1b001dd018e24955b8ecbc7ea9eeacf0 2024-12-10T07:15:52,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/7bbbc2341c0249389ee7a4855ef888a5 is 50, key is test_row_0/B:col10/1733814951616/Put/seqid=0 2024-12-10T07:15:52,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742121_1297 (size=12301) 2024-12-10T07:15:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T07:15:52,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-10T07:15:52,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:52,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:52,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:52,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:52,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:52,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815012264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:52,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815012264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:52,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:52,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:52,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:52,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:52,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:52,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:52,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815012275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:52,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815012277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:52,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:52,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815012277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:52,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:52,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:52,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:52,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-10T07:15:52,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:52,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:52,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:52,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:52,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:52,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T07:15:52,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-10T07:15:52,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:52,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:52,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:52,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:52,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:52,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/7bbbc2341c0249389ee7a4855ef888a5 2024-12-10T07:15:52,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/24562cc497b84d05b8314e769fb7f7cf is 50, key is test_row_0/C:col10/1733814951616/Put/seqid=0 2024-12-10T07:15:52,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742122_1298 (size=12301) 2024-12-10T07:15:52,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-10T07:15:52,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:52,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:52,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:52,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:52,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:52,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:52,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815012775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:52,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:52,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:52,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:52,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815012775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:52,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:52,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:52,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:52,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815012785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:52,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815012788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:52,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-10T07:15:52,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:52,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:52,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:52,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:52,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:52,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:52,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:52,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:52,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815012797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:52,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:52,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:52,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:52,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-10T07:15:52,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:52,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:52,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:52,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:52,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:52,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T07:15:53,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/24562cc497b84d05b8314e769fb7f7cf 2024-12-10T07:15:53,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/1b001dd018e24955b8ecbc7ea9eeacf0 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/1b001dd018e24955b8ecbc7ea9eeacf0 2024-12-10T07:15:53,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/1b001dd018e24955b8ecbc7ea9eeacf0, entries=150, sequenceid=266, filesize=12.0 K 2024-12-10T07:15:53,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/7bbbc2341c0249389ee7a4855ef888a5 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/7bbbc2341c0249389ee7a4855ef888a5 2024-12-10T07:15:53,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/7bbbc2341c0249389ee7a4855ef888a5, entries=150, sequenceid=266, filesize=12.0 K 2024-12-10T07:15:53,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/24562cc497b84d05b8314e769fb7f7cf as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/24562cc497b84d05b8314e769fb7f7cf 2024-12-10T07:15:53,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/24562cc497b84d05b8314e769fb7f7cf, entries=150, sequenceid=266, filesize=12.0 K 2024-12-10T07:15:53,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 07d0f6bb02ffe54509ba6124b6830098 in 1429ms, sequenceid=266, compaction requested=true 2024-12-10T07:15:53,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:53,047 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:53,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:15:53,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:53,048 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:53,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:15:53,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:53,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:15:53,048 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:53,049 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/A is initiating minor compaction (all files) 2024-12-10T07:15:53,049 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/A in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:53,049 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/56610babc4eb4a2db6b9903de51d5a58, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/86f51c63f9cb4051af0f224b13784b0d, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/1b001dd018e24955b8ecbc7ea9eeacf0] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=36.2 K 2024-12-10T07:15:53,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:53,049 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:53,049 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56610babc4eb4a2db6b9903de51d5a58, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733814949121 2024-12-10T07:15:53,049 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/B is initiating minor compaction (all files) 2024-12-10T07:15:53,049 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/B in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:53,049 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/6a136468a02a47c092ab09cd5fe509ba, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/3c4b716a77e943c79e0269e44af76148, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/7bbbc2341c0249389ee7a4855ef888a5] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=36.2 K 2024-12-10T07:15:53,050 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86f51c63f9cb4051af0f224b13784b0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733814949795 2024-12-10T07:15:53,050 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a136468a02a47c092ab09cd5fe509ba, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733814949121 2024-12-10T07:15:53,050 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b001dd018e24955b8ecbc7ea9eeacf0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733814950455 2024-12-10T07:15:53,050 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c4b716a77e943c79e0269e44af76148, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733814949795 2024-12-10T07:15:53,051 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bbbc2341c0249389ee7a4855ef888a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733814950455 2024-12-10T07:15:53,074 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#A#compaction#248 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:53,075 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/53738b0ba8c748bc9c6bc55d0f9e72c0 is 50, key is test_row_0/A:col10/1733814951616/Put/seqid=0 2024-12-10T07:15:53,087 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#B#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:53,088 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/ac6c25645fe04625bab6ece17dd1f6df is 50, key is test_row_0/B:col10/1733814951616/Put/seqid=0 2024-12-10T07:15:53,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742123_1299 (size=12847) 2024-12-10T07:15:53,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-10T07:15:53,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:53,113 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T07:15:53,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:15:53,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:53,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:15:53,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:53,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:15:53,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:53,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/b959d2b2f0cb4fbf94c6a1404a378733 is 50, key is test_row_0/A:col10/1733814951631/Put/seqid=0 2024-12-10T07:15:53,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742124_1300 (size=12847) 2024-12-10T07:15:53,136 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/ac6c25645fe04625bab6ece17dd1f6df as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/ac6c25645fe04625bab6ece17dd1f6df 2024-12-10T07:15:53,143 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/B of 07d0f6bb02ffe54509ba6124b6830098 into ac6c25645fe04625bab6ece17dd1f6df(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:53,143 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:53,143 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/B, priority=13, startTime=1733814953047; duration=0sec 2024-12-10T07:15:53,143 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:53,143 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:B 2024-12-10T07:15:53,143 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:53,145 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:53,145 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/C is initiating minor compaction (all files) 2024-12-10T07:15:53,145 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/C in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:53,145 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/73d4454464aa43778c107034cfb6a279, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/ed32063d07c64e6e997bb5f54bd200b1, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/24562cc497b84d05b8314e769fb7f7cf] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=36.2 K 2024-12-10T07:15:53,146 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 73d4454464aa43778c107034cfb6a279, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733814949121 2024-12-10T07:15:53,146 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting ed32063d07c64e6e997bb5f54bd200b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733814949795 2024-12-10T07:15:53,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742125_1301 (size=12301) 2024-12-10T07:15:53,147 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 24562cc497b84d05b8314e769fb7f7cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733814950455 2024-12-10T07:15:53,148 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/b959d2b2f0cb4fbf94c6a1404a378733 2024-12-10T07:15:53,163 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#C#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:53,163 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/ae5a8aa24e194f2e99a58899c76c2ef2 is 50, key is test_row_0/C:col10/1733814951616/Put/seqid=0 2024-12-10T07:15:53,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/796d1e6edcc346e5b303d4540aac3951 is 50, key is test_row_0/B:col10/1733814951631/Put/seqid=0 2024-12-10T07:15:53,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742126_1302 (size=12847) 2024-12-10T07:15:53,234 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/ae5a8aa24e194f2e99a58899c76c2ef2 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/ae5a8aa24e194f2e99a58899c76c2ef2 2024-12-10T07:15:53,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742127_1303 (size=12301) 2024-12-10T07:15:53,239 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/796d1e6edcc346e5b303d4540aac3951 2024-12-10T07:15:53,243 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/C of 07d0f6bb02ffe54509ba6124b6830098 into ae5a8aa24e194f2e99a58899c76c2ef2(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:53,243 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:53,243 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/C, priority=13, startTime=1733814953048; duration=0sec 2024-12-10T07:15:53,244 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:53,244 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:C 2024-12-10T07:15:53,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/e8b4731fa2434caa841e9c76f2ede618 is 50, key is test_row_0/C:col10/1733814951631/Put/seqid=0 2024-12-10T07:15:53,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742128_1304 (size=12301) 2024-12-10T07:15:53,505 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/53738b0ba8c748bc9c6bc55d0f9e72c0 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/53738b0ba8c748bc9c6bc55d0f9e72c0 2024-12-10T07:15:53,511 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/A of 07d0f6bb02ffe54509ba6124b6830098 into 53738b0ba8c748bc9c6bc55d0f9e72c0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:53,511 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:53,511 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/A, priority=13, startTime=1733814953047; duration=0sec 2024-12-10T07:15:53,511 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:53,511 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:A 2024-12-10T07:15:53,690 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/e8b4731fa2434caa841e9c76f2ede618 2024-12-10T07:15:53,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/b959d2b2f0cb4fbf94c6a1404a378733 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/b959d2b2f0cb4fbf94c6a1404a378733 2024-12-10T07:15:53,711 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/b959d2b2f0cb4fbf94c6a1404a378733, entries=150, sequenceid=281, filesize=12.0 K 2024-12-10T07:15:53,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/796d1e6edcc346e5b303d4540aac3951 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/796d1e6edcc346e5b303d4540aac3951 2024-12-10T07:15:53,716 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/796d1e6edcc346e5b303d4540aac3951, entries=150, sequenceid=281, filesize=12.0 K 2024-12-10T07:15:53,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/e8b4731fa2434caa841e9c76f2ede618 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/e8b4731fa2434caa841e9c76f2ede618 2024-12-10T07:15:53,725 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/e8b4731fa2434caa841e9c76f2ede618, entries=150, sequenceid=281, filesize=12.0 K 2024-12-10T07:15:53,729 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 07d0f6bb02ffe54509ba6124b6830098 in 616ms, sequenceid=281, compaction requested=false 2024-12-10T07:15:53,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:53,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:53,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-10T07:15:53,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-10T07:15:53,747 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-10T07:15:53,747 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8910 sec 2024-12-10T07:15:53,749 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.9190 sec 2024-12-10T07:15:53,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T07:15:53,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:15:53,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:53,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:15:53,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:53,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:15:53,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:53,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:53,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/509f220b2d814b2cbdf8d12eed0970a0 is 50, key is test_row_0/A:col10/1733814953827/Put/seqid=0 2024-12-10T07:15:53,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742129_1305 (size=12301) 2024-12-10T07:15:53,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/509f220b2d814b2cbdf8d12eed0970a0 2024-12-10T07:15:53,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733814953847, max timestamp=1733814953898Num uniques 859; 2024-12-10T07:15:53,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733814953847, max timestamp=1733814953898Num uniques 859; 2024-12-10T07:15:53,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T07:15:53,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733814953847, max timestamp=1733814953898Num uniques 859; 2024-12-10T07:15:53,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T07:15:53,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T07:15:53,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/53b00c42579740888366955c0a883078 is 50, key is test_row_0/B:col10/1733814953827/Put/seqid=0 2024-12-10T07:15:53,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742130_1306 (size=12301) 2024-12-10T07:15:53,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/53b00c42579740888366955c0a883078 2024-12-10T07:15:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-10T07:15:53,989 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:15:53,990 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:15:53,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/ef2a33f78a454debb32ace57bd35184b is 50, key is test_row_0/C:col10/1733814953827/Put/seqid=0 2024-12-10T07:15:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-10T07:15:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T07:15:53,992 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:15:53,993 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:15:53,993 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:15:54,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:54,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815014022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:54,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:54,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815014022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:54,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:54,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:54,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815014023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:54,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:54,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:54,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815014025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:54,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:54,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:54,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742131_1307 (size=12301) 2024-12-10T07:15:54,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815014038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:54,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:54,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/ef2a33f78a454debb32ace57bd35184b 2024-12-10T07:15:54,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/509f220b2d814b2cbdf8d12eed0970a0 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/509f220b2d814b2cbdf8d12eed0970a0 2024-12-10T07:15:54,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T07:15:54,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/509f220b2d814b2cbdf8d12eed0970a0, entries=150, sequenceid=295, filesize=12.0 K 2024-12-10T07:15:54,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/53b00c42579740888366955c0a883078 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/53b00c42579740888366955c0a883078 2024-12-10T07:15:54,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/53b00c42579740888366955c0a883078, entries=150, sequenceid=295, filesize=12.0 K 2024-12-10T07:15:54,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/ef2a33f78a454debb32ace57bd35184b as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/ef2a33f78a454debb32ace57bd35184b 2024-12-10T07:15:54,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/ef2a33f78a454debb32ace57bd35184b, entries=150, sequenceid=295, filesize=12.0 K 2024-12-10T07:15:54,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 07d0f6bb02ffe54509ba6124b6830098 in 303ms, sequenceid=295, compaction requested=true 2024-12-10T07:15:54,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:54,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:15:54,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:54,132 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:54,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:15:54,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:54,133 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:54,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:15:54,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:54,137 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:54,137 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/B is initiating minor compaction (all files) 2024-12-10T07:15:54,138 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/B in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:54,138 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/ac6c25645fe04625bab6ece17dd1f6df, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/796d1e6edcc346e5b303d4540aac3951, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/53b00c42579740888366955c0a883078] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=36.6 K 2024-12-10T07:15:54,138 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting ac6c25645fe04625bab6ece17dd1f6df, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733814950455 2024-12-10T07:15:54,138 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 796d1e6edcc346e5b303d4540aac3951, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733814951628 2024-12-10T07:15:54,139 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 53b00c42579740888366955c0a883078, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733814953797 2024-12-10T07:15:54,141 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:54,141 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/A is initiating minor compaction (all files) 2024-12-10T07:15:54,141 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/A in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:54,141 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/53738b0ba8c748bc9c6bc55d0f9e72c0, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/b959d2b2f0cb4fbf94c6a1404a378733, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/509f220b2d814b2cbdf8d12eed0970a0] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=36.6 K 2024-12-10T07:15:54,141 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53738b0ba8c748bc9c6bc55d0f9e72c0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733814950455 2024-12-10T07:15:54,142 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting b959d2b2f0cb4fbf94c6a1404a378733, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733814951628 2024-12-10T07:15:54,142 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 509f220b2d814b2cbdf8d12eed0970a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733814953797 2024-12-10T07:15:54,153 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-10T07:15:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:54,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T07:15:54,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:54,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:54,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:54,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:54,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:54,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:15:54,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:54,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:15:54,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:54,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:15:54,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:54,163 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#B#compaction#257 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:54,163 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/823a48b873ef42e692c16b11a0660e0a is 50, key is test_row_0/B:col10/1733814953827/Put/seqid=0 2024-12-10T07:15:54,167 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#A#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:54,167 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/a706d063dd0e4db2becb6bc982318c6c is 50, key is test_row_0/A:col10/1733814953827/Put/seqid=0 2024-12-10T07:15:54,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/862aed12aba94a619c3222c43c9d91cd is 50, key is test_row_0/A:col10/1733814954148/Put/seqid=0 2024-12-10T07:15:54,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815014175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:54,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:54,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815014188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:54,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742132_1308 (size=12949) 2024-12-10T07:15:54,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742133_1309 (size=12949) 2024-12-10T07:15:54,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:54,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815014230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:54,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:54,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742134_1310 (size=12301) 2024-12-10T07:15:54,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815014241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:54,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815014245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:54,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:54,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T07:15:54,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T07:15:54,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:54,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:54,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:54,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:54,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:54,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815014334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:54,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:54,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:54,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815014346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:54,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:54,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:54,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815014367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:54,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:54,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:54,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815014375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:54,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:54,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:54,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815014376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:54,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:54,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T07:15:54,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:54,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:54,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:54,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:54,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:54,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815014555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:54,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815014578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:54,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:54,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815014595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815014595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:54,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:54,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815014605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:54,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T07:15:54,652 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/a706d063dd0e4db2becb6bc982318c6c as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/a706d063dd0e4db2becb6bc982318c6c 2024-12-10T07:15:54,652 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/823a48b873ef42e692c16b11a0660e0a as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/823a48b873ef42e692c16b11a0660e0a 2024-12-10T07:15:54,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T07:15:54,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:54,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:54,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:54,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:54,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:54,662 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/B of 07d0f6bb02ffe54509ba6124b6830098 into 823a48b873ef42e692c16b11a0660e0a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:54,662 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/A of 07d0f6bb02ffe54509ba6124b6830098 into a706d063dd0e4db2becb6bc982318c6c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:54,662 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:54,662 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:54,662 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/A, priority=13, startTime=1733814954132; duration=0sec 2024-12-10T07:15:54,662 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/B, priority=13, startTime=1733814954132; duration=0sec 2024-12-10T07:15:54,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:54,662 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:54,662 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:A 2024-12-10T07:15:54,662 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:54,662 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:54,662 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:B 2024-12-10T07:15:54,673 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:54,673 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/C is initiating minor compaction (all files) 2024-12-10T07:15:54,673 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/C in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:54,673 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/ae5a8aa24e194f2e99a58899c76c2ef2, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/e8b4731fa2434caa841e9c76f2ede618, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/ef2a33f78a454debb32ace57bd35184b] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=36.6 K 2024-12-10T07:15:54,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/862aed12aba94a619c3222c43c9d91cd 2024-12-10T07:15:54,674 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae5a8aa24e194f2e99a58899c76c2ef2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733814950455 2024-12-10T07:15:54,675 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8b4731fa2434caa841e9c76f2ede618, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733814951628 2024-12-10T07:15:54,675 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef2a33f78a454debb32ace57bd35184b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733814953797 2024-12-10T07:15:54,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/755b4a2f748745b78557fd6d3a76bc7e is 50, key is test_row_0/B:col10/1733814954148/Put/seqid=0 2024-12-10T07:15:54,739 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#C#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:54,739 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/0c2c994798384c33be14ca5fd1c5c576 is 50, key is test_row_0/C:col10/1733814953827/Put/seqid=0 2024-12-10T07:15:54,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742135_1311 (size=12301) 2024-12-10T07:15:54,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742136_1312 (size=12949) 2024-12-10T07:15:54,815 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/0c2c994798384c33be14ca5fd1c5c576 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/0c2c994798384c33be14ca5fd1c5c576 2024-12-10T07:15:54,823 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/C of 07d0f6bb02ffe54509ba6124b6830098 into 0c2c994798384c33be14ca5fd1c5c576(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:54,824 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:54,824 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/C, priority=13, startTime=1733814954133; duration=0sec 2024-12-10T07:15:54,824 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:54,824 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:C 2024-12-10T07:15:54,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T07:15:54,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:54,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:54,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:54,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:54,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:54,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815014885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:54,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:54,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815014885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:54,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:54,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815014905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:54,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:54,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815014915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:54,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815014915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:54,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T07:15:54,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:54,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:54,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:54,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:54,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T07:15:55,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/755b4a2f748745b78557fd6d3a76bc7e 2024-12-10T07:15:55,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T07:15:55,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:55,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:55,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:55,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:55,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:55,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:55,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/0b1cc90e6453442f88fc7e966e538261 is 50, key is test_row_0/C:col10/1733814954148/Put/seqid=0 2024-12-10T07:15:55,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742137_1313 (size=12301) 2024-12-10T07:15:55,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/0b1cc90e6453442f88fc7e966e538261 2024-12-10T07:15:55,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/862aed12aba94a619c3222c43c9d91cd as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/862aed12aba94a619c3222c43c9d91cd 2024-12-10T07:15:55,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/862aed12aba94a619c3222c43c9d91cd, entries=150, sequenceid=331, filesize=12.0 K 2024-12-10T07:15:55,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/755b4a2f748745b78557fd6d3a76bc7e as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/755b4a2f748745b78557fd6d3a76bc7e 2024-12-10T07:15:55,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/755b4a2f748745b78557fd6d3a76bc7e, entries=150, sequenceid=331, filesize=12.0 K 2024-12-10T07:15:55,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/0b1cc90e6453442f88fc7e966e538261 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/0b1cc90e6453442f88fc7e966e538261 2024-12-10T07:15:55,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T07:15:55,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:55,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:55,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:55,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:55,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:55,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:55,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/0b1cc90e6453442f88fc7e966e538261, entries=150, sequenceid=331, filesize=12.0 K 2024-12-10T07:15:55,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 07d0f6bb02ffe54509ba6124b6830098 in 1211ms, sequenceid=331, compaction requested=false 2024-12-10T07:15:55,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:55,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T07:15:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:55,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:15:55,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:55,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:15:55,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:55,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:15:55,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:55,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/2bdccafd4d42469ca3ac55d9443215ae is 50, key is test_row_0/A:col10/1733814954203/Put/seqid=0 2024-12-10T07:15:55,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742138_1314 (size=12301) 2024-12-10T07:15:55,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/2bdccafd4d42469ca3ac55d9443215ae 2024-12-10T07:15:55,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/acdade7993814c6f8dd23ab43cd6d909 is 50, key is test_row_0/B:col10/1733814954203/Put/seqid=0 2024-12-10T07:15:55,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814955475, max timestamp=1733814955497Num uniques 859; 2024-12-10T07:15:55,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:15:55,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814955475, max timestamp=1733814955497Num uniques 859; 2024-12-10T07:15:55,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814955475, max timestamp=1733814955497Num uniques 859; 2024-12-10T07:15:55,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:15:55,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:15:55,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T07:15:55,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:55,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:55,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:55,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:55,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:55,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742139_1315 (size=12301) 2024-12-10T07:15:55,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815015544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:55,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:55,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815015558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:55,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815015559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:55,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815015562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:55,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815015563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:55,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:55,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T07:15:55,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:55,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:55,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:55,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:55,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:55,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815015675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:55,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815015685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:55,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815015685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:55,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815015685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:55,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:55,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815015686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:55,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:55,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T07:15:55,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:55,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:55,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:55,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:55,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:55,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815015884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:55,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815015897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:55,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815015897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:55,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:55,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815015900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:55,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:55,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815015906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:55,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:55,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:55,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/acdade7993814c6f8dd23ab43cd6d909 2024-12-10T07:15:55,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/59b0f0f645b9447fb0dd7f6b40a596f2 is 50, key is test_row_0/C:col10/1733814954203/Put/seqid=0 2024-12-10T07:15:55,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T07:15:55,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:55,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:55,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:55,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:55,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:56,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742140_1316 (size=12301) 2024-12-10T07:15:56,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/59b0f0f645b9447fb0dd7f6b40a596f2 2024-12-10T07:15:56,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/2bdccafd4d42469ca3ac55d9443215ae as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/2bdccafd4d42469ca3ac55d9443215ae 2024-12-10T07:15:56,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/2bdccafd4d42469ca3ac55d9443215ae, entries=150, sequenceid=345, filesize=12.0 K 2024-12-10T07:15:56,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/acdade7993814c6f8dd23ab43cd6d909 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/acdade7993814c6f8dd23ab43cd6d909 2024-12-10T07:15:56,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/acdade7993814c6f8dd23ab43cd6d909, entries=150, sequenceid=345, filesize=12.0 K 2024-12-10T07:15:56,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/59b0f0f645b9447fb0dd7f6b40a596f2 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/59b0f0f645b9447fb0dd7f6b40a596f2 2024-12-10T07:15:56,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/59b0f0f645b9447fb0dd7f6b40a596f2, entries=150, sequenceid=345, filesize=12.0 K 2024-12-10T07:15:56,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 07d0f6bb02ffe54509ba6124b6830098 in 639ms, sequenceid=345, compaction requested=true 2024-12-10T07:15:56,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:56,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:15:56,049 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:56,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:56,049 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:56,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:15:56,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:56,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:15:56,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:56,053 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:56,053 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:56,053 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/A is initiating minor compaction (all files) 2024-12-10T07:15:56,053 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/B is initiating minor compaction (all files) 2024-12-10T07:15:56,053 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/B in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:56,053 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/823a48b873ef42e692c16b11a0660e0a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/755b4a2f748745b78557fd6d3a76bc7e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/acdade7993814c6f8dd23ab43cd6d909] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=36.7 K 2024-12-10T07:15:56,053 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/A in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:56,053 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/a706d063dd0e4db2becb6bc982318c6c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/862aed12aba94a619c3222c43c9d91cd, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/2bdccafd4d42469ca3ac55d9443215ae] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=36.7 K 2024-12-10T07:15:56,054 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 823a48b873ef42e692c16b11a0660e0a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733814953797 2024-12-10T07:15:56,057 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting a706d063dd0e4db2becb6bc982318c6c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733814953797 2024-12-10T07:15:56,057 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 755b4a2f748745b78557fd6d3a76bc7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733814954022 2024-12-10T07:15:56,058 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 862aed12aba94a619c3222c43c9d91cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733814954022 2024-12-10T07:15:56,059 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting acdade7993814c6f8dd23ab43cd6d909, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733814954203 2024-12-10T07:15:56,061 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bdccafd4d42469ca3ac55d9443215ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733814954203 2024-12-10T07:15:56,074 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#B#compaction#266 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:56,075 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/2d68cd45f6544b8cb45fea661d2d3268 is 50, key is test_row_0/B:col10/1733814954203/Put/seqid=0 2024-12-10T07:15:56,087 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#A#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:56,088 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/2d0384138f994e9caec0cdbeb3556e05 is 50, key is test_row_0/A:col10/1733814954203/Put/seqid=0 2024-12-10T07:15:56,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742141_1317 (size=13051) 2024-12-10T07:15:56,124 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/2d68cd45f6544b8cb45fea661d2d3268 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/2d68cd45f6544b8cb45fea661d2d3268 2024-12-10T07:15:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T07:15:56,138 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/B of 07d0f6bb02ffe54509ba6124b6830098 into 2d68cd45f6544b8cb45fea661d2d3268(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:56,138 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:56,138 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/B, priority=13, startTime=1733814956049; duration=0sec 2024-12-10T07:15:56,139 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:56,139 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:B 2024-12-10T07:15:56,139 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:56,140 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:56,140 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/C is initiating minor compaction (all files) 2024-12-10T07:15:56,140 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/C in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:56,140 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/0c2c994798384c33be14ca5fd1c5c576, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/0b1cc90e6453442f88fc7e966e538261, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/59b0f0f645b9447fb0dd7f6b40a596f2] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=36.7 K 2024-12-10T07:15:56,141 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c2c994798384c33be14ca5fd1c5c576, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733814953797 2024-12-10T07:15:56,142 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b1cc90e6453442f88fc7e966e538261, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733814954022 2024-12-10T07:15:56,143 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 59b0f0f645b9447fb0dd7f6b40a596f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733814954203 2024-12-10T07:15:56,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742142_1318 (size=13051) 2024-12-10T07:15:56,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-10T07:15:56,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:56,157 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-10T07:15:56,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:15:56,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:56,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:15:56,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:56,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:15:56,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:56,165 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#C#compaction#268 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:56,166 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/355f36b697964ca5b233e2cd3508be89 is 50, key is test_row_0/C:col10/1733814954203/Put/seqid=0 2024-12-10T07:15:56,170 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/2d0384138f994e9caec0cdbeb3556e05 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/2d0384138f994e9caec0cdbeb3556e05 2024-12-10T07:15:56,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/969970a436cf4f22b49306df5d7bdd9c is 50, key is test_row_0/A:col10/1733814955558/Put/seqid=0 2024-12-10T07:15:56,191 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/A of 07d0f6bb02ffe54509ba6124b6830098 into 2d0384138f994e9caec0cdbeb3556e05(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:56,191 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:56,191 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/A, priority=13, startTime=1733814956049; duration=0sec 2024-12-10T07:15:56,191 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:56,191 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:A 2024-12-10T07:15:56,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:56,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742143_1319 (size=13051) 2024-12-10T07:15:56,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815016237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:56,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:56,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815016239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:56,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:56,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742144_1320 (size=12301) 2024-12-10T07:15:56,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:56,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815016242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:56,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815016246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:56,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815016246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:56,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:56,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:56,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:56,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815016355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:56,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:56,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:56,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815016355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:56,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:56,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:56,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815016374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:56,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:56,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:56,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815016375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:56,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:56,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:56,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815016374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:56,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:56,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:56,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815016565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:56,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:56,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:56,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815016575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:56,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:56,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815016595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:56,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815016595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:56,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815016595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:56,641 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/355f36b697964ca5b233e2cd3508be89 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/355f36b697964ca5b233e2cd3508be89 2024-12-10T07:15:56,646 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/969970a436cf4f22b49306df5d7bdd9c 2024-12-10T07:15:56,664 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/C of 07d0f6bb02ffe54509ba6124b6830098 into 355f36b697964ca5b233e2cd3508be89(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:56,664 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:56,664 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/C, priority=13, startTime=1733814956050; duration=0sec 2024-12-10T07:15:56,664 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:56,664 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:C 2024-12-10T07:15:56,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/5faca50c3b824eb1802ce9363ecde377 is 50, key is test_row_0/B:col10/1733814955558/Put/seqid=0 2024-12-10T07:15:56,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742145_1321 (size=12301) 2024-12-10T07:15:56,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:56,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815016875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:56,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:56,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:56,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815016900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:56,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:56,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:56,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815016915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:56,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:56,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:56,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815016915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:56,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:56,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815016917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:57,122 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/5faca50c3b824eb1802ce9363ecde377 2024-12-10T07:15:57,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/743faaa3207c43c885e6ef96ea402f14 is 50, key is test_row_0/C:col10/1733814955558/Put/seqid=0 2024-12-10T07:15:57,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742146_1322 (size=12301) 2024-12-10T07:15:57,175 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/743faaa3207c43c885e6ef96ea402f14 2024-12-10T07:15:57,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/969970a436cf4f22b49306df5d7bdd9c as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/969970a436cf4f22b49306df5d7bdd9c 2024-12-10T07:15:57,197 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/969970a436cf4f22b49306df5d7bdd9c, entries=150, sequenceid=382, filesize=12.0 K 2024-12-10T07:15:57,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/5faca50c3b824eb1802ce9363ecde377 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/5faca50c3b824eb1802ce9363ecde377 2024-12-10T07:15:57,212 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/5faca50c3b824eb1802ce9363ecde377, entries=150, sequenceid=382, filesize=12.0 K 2024-12-10T07:15:57,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/743faaa3207c43c885e6ef96ea402f14 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/743faaa3207c43c885e6ef96ea402f14 2024-12-10T07:15:57,219 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/743faaa3207c43c885e6ef96ea402f14, entries=150, sequenceid=382, filesize=12.0 K 2024-12-10T07:15:57,221 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 07d0f6bb02ffe54509ba6124b6830098 in 1064ms, sequenceid=382, compaction requested=false 2024-12-10T07:15:57,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:57,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:57,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-10T07:15:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-10T07:15:57,230 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-10T07:15:57,230 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2320 sec 2024-12-10T07:15:57,233 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 3.2410 sec 2024-12-10T07:15:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:57,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T07:15:57,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:15:57,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:57,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:15:57,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:57,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:15:57,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:57,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/952e308b617a4fe981b175dd6fd6ec39 is 50, key is test_row_0/A:col10/1733814956240/Put/seqid=0 2024-12-10T07:15:57,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742147_1323 (size=12301) 2024-12-10T07:15:57,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815017428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:57,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815017429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:57,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:57,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815017429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:57,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733814957418, max timestamp=1733814957432Num uniques 859; 2024-12-10T07:15:57,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T07:15:57,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733814957418, max timestamp=1733814957432Num uniques 859; 2024-12-10T07:15:57,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733814957418, max timestamp=1733814957432Num uniques 859; 2024-12-10T07:15:57,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T07:15:57,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-10T07:15:57,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:57,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815017460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:57,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:57,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815017461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:57,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:57,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:57,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:57,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815017544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:57,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:57,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:57,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815017544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:57,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:57,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:57,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815017544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:57,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:57,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815017575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:57,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:57,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815017575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:57,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:57,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:57,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815017754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:57,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:57,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:57,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815017755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:57,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:57,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:57,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815017755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:57,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:57,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:57,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815017784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:57,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:57,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:57,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815017784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:57,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:57,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:57,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/952e308b617a4fe981b175dd6fd6ec39 2024-12-10T07:15:57,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/7e097cb9dc7c4ffda1cf7b1bba144096 is 50, key is test_row_0/B:col10/1733814956240/Put/seqid=0 2024-12-10T07:15:57,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742148_1324 (size=12301) 2024-12-10T07:15:57,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/7e097cb9dc7c4ffda1cf7b1bba144096 2024-12-10T07:15:57,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/40eccb25fe344ca19aa454c3d31d4169 is 50, key is test_row_0/C:col10/1733814956240/Put/seqid=0 2024-12-10T07:15:57,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742149_1325 (size=12301) 2024-12-10T07:15:58,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:58,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815018064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:58,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815018064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:58,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815018064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:58,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:58,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815018095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:58,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:58,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815018095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:58,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:58,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:58,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-10T07:15:58,155 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:15:58,156 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:15:58,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-10T07:15:58,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-10T07:15:58,158 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:15:58,158 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:15:58,158 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:15:58,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/40eccb25fe344ca19aa454c3d31d4169 2024-12-10T07:15:58,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/952e308b617a4fe981b175dd6fd6ec39 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/952e308b617a4fe981b175dd6fd6ec39 2024-12-10T07:15:58,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/952e308b617a4fe981b175dd6fd6ec39, entries=150, sequenceid=395, filesize=12.0 K 2024-12-10T07:15:58,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/7e097cb9dc7c4ffda1cf7b1bba144096 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/7e097cb9dc7c4ffda1cf7b1bba144096 2024-12-10T07:15:58,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/7e097cb9dc7c4ffda1cf7b1bba144096, entries=150, sequenceid=395, filesize=12.0 K 2024-12-10T07:15:58,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/40eccb25fe344ca19aa454c3d31d4169 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/40eccb25fe344ca19aa454c3d31d4169 2024-12-10T07:15:58,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/40eccb25fe344ca19aa454c3d31d4169, entries=150, sequenceid=395, filesize=12.0 K 2024-12-10T07:15:58,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 07d0f6bb02ffe54509ba6124b6830098 in 850ms, sequenceid=395, compaction requested=true 2024-12-10T07:15:58,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:58,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:15:58,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:58,237 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:58,237 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:58,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:15:58,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:58,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:15:58,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:58,238 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:58,238 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:58,238 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/A is initiating minor compaction (all files) 2024-12-10T07:15:58,238 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/B is initiating minor compaction (all files) 2024-12-10T07:15:58,238 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/B in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:58,238 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/A in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:58,238 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/2d68cd45f6544b8cb45fea661d2d3268, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/5faca50c3b824eb1802ce9363ecde377, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/7e097cb9dc7c4ffda1cf7b1bba144096] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=36.8 K 2024-12-10T07:15:58,238 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/2d0384138f994e9caec0cdbeb3556e05, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/969970a436cf4f22b49306df5d7bdd9c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/952e308b617a4fe981b175dd6fd6ec39] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=36.8 K 2024-12-10T07:15:58,239 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d0384138f994e9caec0cdbeb3556e05, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733814954203 2024-12-10T07:15:58,239 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d68cd45f6544b8cb45fea661d2d3268, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733814954203 2024-12-10T07:15:58,239 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 969970a436cf4f22b49306df5d7bdd9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733814955546 2024-12-10T07:15:58,239 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 5faca50c3b824eb1802ce9363ecde377, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733814955546 2024-12-10T07:15:58,239 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e097cb9dc7c4ffda1cf7b1bba144096, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733814956229 2024-12-10T07:15:58,239 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 952e308b617a4fe981b175dd6fd6ec39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733814956229 2024-12-10T07:15:58,245 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#A#compaction#275 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:58,245 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#B#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:58,245 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/220b8f60f0fa4da0aab2ca15249e807e is 50, key is test_row_0/A:col10/1733814956240/Put/seqid=0 2024-12-10T07:15:58,246 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/bc44fa3426054e30b6fbdc11735e931d is 50, key is test_row_0/B:col10/1733814956240/Put/seqid=0 2024-12-10T07:15:58,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742150_1326 (size=13153) 2024-12-10T07:15:58,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742151_1327 (size=13153) 2024-12-10T07:15:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-10T07:15:58,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-10T07:15:58,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:58,313 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-10T07:15:58,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:15:58,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:58,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:15:58,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:58,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:15:58,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:15:58,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/6a2d90e60765431faa1548240f6e3260 is 50, key is test_row_0/A:col10/1733814957460/Put/seqid=0 2024-12-10T07:15:58,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742152_1328 (size=12301) 2024-12-10T07:15:58,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-10T07:15:58,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:58,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:58,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:58,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815018586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:58,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:58,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:58,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815018591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:58,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:58,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:58,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815018591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:58,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:58,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:58,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815018604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:58,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:58,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815018604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:58,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:58,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:58,653 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/220b8f60f0fa4da0aab2ca15249e807e as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/220b8f60f0fa4da0aab2ca15249e807e 2024-12-10T07:15:58,653 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/bc44fa3426054e30b6fbdc11735e931d as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/bc44fa3426054e30b6fbdc11735e931d 2024-12-10T07:15:58,657 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/A of 07d0f6bb02ffe54509ba6124b6830098 into 220b8f60f0fa4da0aab2ca15249e807e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:58,657 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/B of 07d0f6bb02ffe54509ba6124b6830098 into bc44fa3426054e30b6fbdc11735e931d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:58,657 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:58,657 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:58,657 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/B, priority=13, startTime=1733814958237; duration=0sec 2024-12-10T07:15:58,657 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/A, priority=13, startTime=1733814958237; duration=0sec 2024-12-10T07:15:58,657 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:58,657 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:58,657 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:B 2024-12-10T07:15:58,657 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:A 2024-12-10T07:15:58,657 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:58,658 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:58,658 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/C is initiating minor compaction (all files) 2024-12-10T07:15:58,658 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/C in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:58,658 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/355f36b697964ca5b233e2cd3508be89, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/743faaa3207c43c885e6ef96ea402f14, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/40eccb25fe344ca19aa454c3d31d4169] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=36.8 K 2024-12-10T07:15:58,659 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 355f36b697964ca5b233e2cd3508be89, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733814954203 2024-12-10T07:15:58,659 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 743faaa3207c43c885e6ef96ea402f14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733814955546 2024-12-10T07:15:58,659 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 40eccb25fe344ca19aa454c3d31d4169, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733814956229 2024-12-10T07:15:58,664 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#C#compaction#278 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:58,665 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/73ca0abe3c3746e19272b9b1663a515a is 50, key is test_row_0/C:col10/1733814956240/Put/seqid=0 2024-12-10T07:15:58,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742153_1329 (size=13153) 2024-12-10T07:15:58,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:58,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815018694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:58,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:58,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815018704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:58,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:58,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815018704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:58,722 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/6a2d90e60765431faa1548240f6e3260 2024-12-10T07:15:58,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/acefc8453a4e4f3893efa81bb724b31c is 50, key is test_row_0/B:col10/1733814957460/Put/seqid=0 2024-12-10T07:15:58,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742154_1330 (size=12301) 2024-12-10T07:15:58,732 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/acefc8453a4e4f3893efa81bb724b31c 2024-12-10T07:15:58,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/6cd333e0cd4f4555ac34fb09117207a5 is 50, key is test_row_0/C:col10/1733814957460/Put/seqid=0 2024-12-10T07:15:58,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742155_1331 (size=12301) 2024-12-10T07:15:58,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-10T07:15:58,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:58,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815018904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:58,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:58,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815018914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:58,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:58,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815018914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:58,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:58,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:59,075 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/73ca0abe3c3746e19272b9b1663a515a as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/73ca0abe3c3746e19272b9b1663a515a 2024-12-10T07:15:59,081 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/C of 07d0f6bb02ffe54509ba6124b6830098 into 73ca0abe3c3746e19272b9b1663a515a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:59,081 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:59,081 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/C, priority=13, startTime=1733814958238; duration=0sec 2024-12-10T07:15:59,081 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:59,081 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:C 2024-12-10T07:15:59,143 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/6cd333e0cd4f4555ac34fb09117207a5 2024-12-10T07:15:59,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/6a2d90e60765431faa1548240f6e3260 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/6a2d90e60765431faa1548240f6e3260 2024-12-10T07:15:59,154 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/6a2d90e60765431faa1548240f6e3260, entries=150, sequenceid=429, filesize=12.0 K 2024-12-10T07:15:59,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/acefc8453a4e4f3893efa81bb724b31c as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/acefc8453a4e4f3893efa81bb724b31c 2024-12-10T07:15:59,162 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/acefc8453a4e4f3893efa81bb724b31c, entries=150, sequenceid=429, filesize=12.0 K 2024-12-10T07:15:59,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/6cd333e0cd4f4555ac34fb09117207a5 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/6cd333e0cd4f4555ac34fb09117207a5 2024-12-10T07:15:59,170 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/6cd333e0cd4f4555ac34fb09117207a5, entries=150, sequenceid=429, filesize=12.0 K 2024-12-10T07:15:59,171 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=60.38 KB/61830 for 07d0f6bb02ffe54509ba6124b6830098 in 857ms, sequenceid=429, compaction requested=false 2024-12-10T07:15:59,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:59,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:59,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-10T07:15:59,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-10T07:15:59,176 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-10T07:15:59,177 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0140 sec 2024-12-10T07:15:59,181 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.0220 sec 2024-12-10T07:15:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:59,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T07:15:59,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:15:59,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:59,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:15:59,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:59,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:15:59,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:59,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/311b2356ecce46ee84961ce7c36d1a98 is 50, key is test_row_0/A:col10/1733814958584/Put/seqid=0 2024-12-10T07:15:59,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742156_1332 (size=12301) 2024-12-10T07:15:59,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/311b2356ecce46ee84961ce7c36d1a98 2024-12-10T07:15:59,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/aab45fa065384ad3a070866d34ee8feb is 50, key is test_row_0/B:col10/1733814958584/Put/seqid=0 2024-12-10T07:15:59,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742157_1333 (size=12301) 2024-12-10T07:15:59,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815019256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:59,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815019258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:59,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:59,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815019267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:59,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-10T07:15:59,295 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:15:59,295 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:15:59,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-10T07:15:59,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-10T07:15:59,297 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:15:59,297 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:15:59,297 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:15:59,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:59,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815019364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:59,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:59,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815019364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:59,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:59,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:59,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:59,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815019374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:59,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-10T07:15:59,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-10T07:15:59,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:59,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:59,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:59,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:59,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:59,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815019575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:59,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:59,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815019575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:59,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:59,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:59,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:59,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815019584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:59,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:59,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-10T07:15:59,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:59,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:59,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:59,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:59,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:59,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:15:59,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-10T07:15:59,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:59,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815019615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:59,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:59,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815019615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:59,617 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 07d0f6bb02ffe54509ba6124b6830098 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:15:59,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,617 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 07d0f6bb02ffe54509ba6124b6830098 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:15:59,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:59,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:59,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/aab45fa065384ad3a070866d34ee8feb 2024-12-10T07:15:59,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/daee1449ecf343f3a483c7c7069e2fe0 is 50, key is test_row_0/C:col10/1733814958584/Put/seqid=0 2024-12-10T07:15:59,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742158_1334 (size=12301) 2024-12-10T07:15:59,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/daee1449ecf343f3a483c7c7069e2fe0 2024-12-10T07:15:59,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/311b2356ecce46ee84961ce7c36d1a98 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/311b2356ecce46ee84961ce7c36d1a98 2024-12-10T07:15:59,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/311b2356ecce46ee84961ce7c36d1a98, entries=150, sequenceid=445, filesize=12.0 K 2024-12-10T07:15:59,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/aab45fa065384ad3a070866d34ee8feb as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/aab45fa065384ad3a070866d34ee8feb 2024-12-10T07:15:59,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/aab45fa065384ad3a070866d34ee8feb, entries=150, sequenceid=445, filesize=12.0 K 2024-12-10T07:15:59,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/daee1449ecf343f3a483c7c7069e2fe0 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/daee1449ecf343f3a483c7c7069e2fe0 2024-12-10T07:15:59,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/daee1449ecf343f3a483c7c7069e2fe0, entries=150, sequenceid=445, filesize=12.0 K 2024-12-10T07:15:59,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 07d0f6bb02ffe54509ba6124b6830098 in 466ms, sequenceid=445, compaction requested=true 2024-12-10T07:15:59,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:59,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:15:59,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:59,684 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:59,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:15:59,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:59,684 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:59,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:15:59,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:59,685 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:59,685 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/B is initiating minor compaction (all files) 2024-12-10T07:15:59,685 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/B in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:59,685 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/bc44fa3426054e30b6fbdc11735e931d, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/acefc8453a4e4f3893efa81bb724b31c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/aab45fa065384ad3a070866d34ee8feb] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=36.9 K 2024-12-10T07:15:59,685 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:59,685 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/A is initiating minor compaction (all files) 2024-12-10T07:15:59,685 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/A in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:59,685 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/220b8f60f0fa4da0aab2ca15249e807e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/6a2d90e60765431faa1548240f6e3260, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/311b2356ecce46ee84961ce7c36d1a98] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=36.9 K 2024-12-10T07:15:59,686 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting bc44fa3426054e30b6fbdc11735e931d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733814956229 2024-12-10T07:15:59,686 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 220b8f60f0fa4da0aab2ca15249e807e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733814956229 2024-12-10T07:15:59,686 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting acefc8453a4e4f3893efa81bb724b31c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733814957452 2024-12-10T07:15:59,686 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a2d90e60765431faa1548240f6e3260, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733814957452 2024-12-10T07:15:59,686 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting aab45fa065384ad3a070866d34ee8feb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1733814958584 2024-12-10T07:15:59,686 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 311b2356ecce46ee84961ce7c36d1a98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1733814958584 2024-12-10T07:15:59,692 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#B#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:59,692 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#A#compaction#284 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:59,693 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/fdfea959063c416b8f2bf4391df21461 is 50, key is test_row_0/A:col10/1733814958584/Put/seqid=0 2024-12-10T07:15:59,693 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/9bf62b6517314638b798a56af3b61d14 is 50, key is test_row_0/B:col10/1733814958584/Put/seqid=0 2024-12-10T07:15:59,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742160_1336 (size=13255) 2024-12-10T07:15:59,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742159_1335 (size=13255) 2024-12-10T07:15:59,702 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/fdfea959063c416b8f2bf4391df21461 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/fdfea959063c416b8f2bf4391df21461 2024-12-10T07:15:59,706 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/A of 07d0f6bb02ffe54509ba6124b6830098 into fdfea959063c416b8f2bf4391df21461(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:59,706 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:59,706 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/A, priority=13, startTime=1733814959684; duration=0sec 2024-12-10T07:15:59,706 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:15:59,706 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:A 2024-12-10T07:15:59,706 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:15:59,707 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:15:59,707 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/C is initiating minor compaction (all files) 2024-12-10T07:15:59,707 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/C in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:59,707 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/73ca0abe3c3746e19272b9b1663a515a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/6cd333e0cd4f4555ac34fb09117207a5, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/daee1449ecf343f3a483c7c7069e2fe0] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=36.9 K 2024-12-10T07:15:59,707 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73ca0abe3c3746e19272b9b1663a515a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733814956229 2024-12-10T07:15:59,708 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cd333e0cd4f4555ac34fb09117207a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733814957452 2024-12-10T07:15:59,708 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting daee1449ecf343f3a483c7c7069e2fe0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1733814958584 2024-12-10T07:15:59,715 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#C#compaction#286 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:15:59,716 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/27b8e7a6c3e943aaa38c524601cd5e71 is 50, key is test_row_0/C:col10/1733814958584/Put/seqid=0 2024-12-10T07:15:59,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742161_1337 (size=13255) 2024-12-10T07:15:59,727 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/27b8e7a6c3e943aaa38c524601cd5e71 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/27b8e7a6c3e943aaa38c524601cd5e71 2024-12-10T07:15:59,731 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/C of 07d0f6bb02ffe54509ba6124b6830098 into 27b8e7a6c3e943aaa38c524601cd5e71(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:15:59,731 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:15:59,731 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/C, priority=13, startTime=1733814959684; duration=0sec 2024-12-10T07:15:59,732 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:15:59,732 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:C 2024-12-10T07:15:59,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-10T07:15:59,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:15:59,756 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T07:15:59,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:15:59,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:59,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:15:59,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:59,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:15:59,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:15:59,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/e44a276637cd4d19a030da4cd996aaa1 is 50, key is test_row_0/A:col10/1733814959247/Put/seqid=0 2024-12-10T07:15:59,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742162_1338 (size=12301) 2024-12-10T07:15:59,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:15:59,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:15:59,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815019900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:59,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815019900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:59,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:59,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:15:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815019904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:15:59,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:15:59,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:15:59,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-10T07:16:00,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815020014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:00,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815020015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:00,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:00,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:00,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:00,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:00,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:00,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:00,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815020015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:00,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:00,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:00,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:00,102 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/9bf62b6517314638b798a56af3b61d14 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/9bf62b6517314638b798a56af3b61d14 2024-12-10T07:16:00,107 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/B of 07d0f6bb02ffe54509ba6124b6830098 into 9bf62b6517314638b798a56af3b61d14(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:00,107 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:16:00,107 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/B, priority=13, startTime=1733814959684; duration=0sec 2024-12-10T07:16:00,107 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:00,107 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:B 2024-12-10T07:16:00,167 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/e44a276637cd4d19a030da4cd996aaa1 2024-12-10T07:16:00,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/e30d731ec456494d8c90f6a6f2140e3e is 50, key is test_row_0/B:col10/1733814959247/Put/seqid=0 2024-12-10T07:16:00,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742163_1339 (size=12301) 2024-12-10T07:16:00,194 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/e30d731ec456494d8c90f6a6f2140e3e 2024-12-10T07:16:00,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/1a98c2201ce8465a981527a98b8dd6c4 is 50, key is test_row_0/C:col10/1733814959247/Put/seqid=0 2024-12-10T07:16:00,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742164_1340 (size=12301) 2024-12-10T07:16:00,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815020224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:00,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:00,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815020224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:00,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:00,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:00,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:00,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815020224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-10T07:16:00,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:00,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815020535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:00,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:00,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:00,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815020535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:00,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815020535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:00,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:00,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:00,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:00,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:00,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:00,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:00,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:00,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:00,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:00,604 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/1a98c2201ce8465a981527a98b8dd6c4 2024-12-10T07:16:00,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/e44a276637cd4d19a030da4cd996aaa1 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/e44a276637cd4d19a030da4cd996aaa1 2024-12-10T07:16:00,616 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/e44a276637cd4d19a030da4cd996aaa1, entries=150, sequenceid=466, filesize=12.0 K 2024-12-10T07:16:00,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/e30d731ec456494d8c90f6a6f2140e3e as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/e30d731ec456494d8c90f6a6f2140e3e 2024-12-10T07:16:00,624 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/e30d731ec456494d8c90f6a6f2140e3e, entries=150, sequenceid=466, filesize=12.0 K 2024-12-10T07:16:00,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/1a98c2201ce8465a981527a98b8dd6c4 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/1a98c2201ce8465a981527a98b8dd6c4 2024-12-10T07:16:00,630 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/1a98c2201ce8465a981527a98b8dd6c4, entries=150, sequenceid=466, filesize=12.0 K 2024-12-10T07:16:00,632 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 07d0f6bb02ffe54509ba6124b6830098 in 876ms, sequenceid=466, compaction requested=false 2024-12-10T07:16:00,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:16:00,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:16:00,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-10T07:16:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-10T07:16:00,636 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-10T07:16:00,636 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3360 sec 2024-12-10T07:16:00,638 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.3410 sec 2024-12-10T07:16:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:16:01,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T07:16:01,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:16:01,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:01,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:16:01,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:01,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:16:01,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:01,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/224e67963d404caea4e653a0609d00a9 is 50, key is test_row_0/A:col10/1733814961045/Put/seqid=0 2024-12-10T07:16:01,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742165_1341 (size=12301) 2024-12-10T07:16:01,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/224e67963d404caea4e653a0609d00a9 2024-12-10T07:16:01,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:01,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815021081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:01,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:01,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815021086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:01,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:01,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:01,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815021090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:01,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:01,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/c5be785f46d349828812b5fa772938a7 is 50, key is test_row_0/B:col10/1733814961045/Put/seqid=0 2024-12-10T07:16:01,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742166_1342 (size=12301) 2024-12-10T07:16:01,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/c5be785f46d349828812b5fa772938a7 2024-12-10T07:16:01,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/a16ec531d54746a0a955a84e2a2a7bb1 is 50, key is test_row_0/C:col10/1733814961045/Put/seqid=0 2024-12-10T07:16:01,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742167_1343 (size=12301) 2024-12-10T07:16:01,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/a16ec531d54746a0a955a84e2a2a7bb1 2024-12-10T07:16:01,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/224e67963d404caea4e653a0609d00a9 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/224e67963d404caea4e653a0609d00a9 2024-12-10T07:16:01,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/224e67963d404caea4e653a0609d00a9, entries=150, sequenceid=481, filesize=12.0 K 2024-12-10T07:16:01,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/c5be785f46d349828812b5fa772938a7 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/c5be785f46d349828812b5fa772938a7 2024-12-10T07:16:01,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/c5be785f46d349828812b5fa772938a7, entries=150, sequenceid=481, filesize=12.0 K 2024-12-10T07:16:01,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/a16ec531d54746a0a955a84e2a2a7bb1 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/a16ec531d54746a0a955a84e2a2a7bb1 2024-12-10T07:16:01,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/a16ec531d54746a0a955a84e2a2a7bb1, entries=150, sequenceid=481, filesize=12.0 K 2024-12-10T07:16:01,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 07d0f6bb02ffe54509ba6124b6830098 in 145ms, sequenceid=481, compaction requested=true 2024-12-10T07:16:01,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:16:01,193 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:16:01,194 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:16:01,194 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/A is initiating minor compaction (all files) 2024-12-10T07:16:01,194 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/A in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:16:01,194 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/fdfea959063c416b8f2bf4391df21461, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/e44a276637cd4d19a030da4cd996aaa1, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/224e67963d404caea4e653a0609d00a9] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=37.0 K 2024-12-10T07:16:01,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:16:01,195 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdfea959063c416b8f2bf4391df21461, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1733814958584 2024-12-10T07:16:01,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:01,195 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:16:01,195 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting e44a276637cd4d19a030da4cd996aaa1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1733814959230 2024-12-10T07:16:01,196 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 224e67963d404caea4e653a0609d00a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733814959898 2024-12-10T07:16:01,196 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:16:01,197 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/B is initiating minor compaction (all files) 2024-12-10T07:16:01,197 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/B in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:16:01,197 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/9bf62b6517314638b798a56af3b61d14, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/e30d731ec456494d8c90f6a6f2140e3e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/c5be785f46d349828812b5fa772938a7] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=37.0 K 2024-12-10T07:16:01,198 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bf62b6517314638b798a56af3b61d14, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1733814958584 2024-12-10T07:16:01,199 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting e30d731ec456494d8c90f6a6f2140e3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1733814959230 2024-12-10T07:16:01,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:16:01,200 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting c5be785f46d349828812b5fa772938a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733814959898 2024-12-10T07:16:01,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:16:01,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:01,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:16:01,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:16:01,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T07:16:01,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814961070, max timestamp=1733814961208Num uniques 859; 2024-12-10T07:16:01,212 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#A#compaction#293 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:01,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:16:01,212 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/704f14f771b44af194e253bcc7b53af1 is 50, key is test_row_0/A:col10/1733814961045/Put/seqid=0 2024-12-10T07:16:01,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:16:01,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:16:01,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:16:01,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:16:01,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:16:01,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:16:01,213 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#B#compaction#294 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:01,214 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/3102ceb6a6864aea9ac74f5f99ceca4f is 50, key is test_row_0/B:col10/1733814961045/Put/seqid=0 2024-12-10T07:16:01,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742169_1345 (size=13357) 2024-12-10T07:16:01,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/7ab7cc2bf9084c19bf7e7d615c67ac5e is 50, key is test_row_0/A:col10/1733814961208/Put/seqid=0 2024-12-10T07:16:01,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742170_1346 (size=17181) 2024-12-10T07:16:01,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742168_1344 (size=13357) 2024-12-10T07:16:01,250 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/704f14f771b44af194e253bcc7b53af1 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/704f14f771b44af194e253bcc7b53af1 2024-12-10T07:16:01,257 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/A of 07d0f6bb02ffe54509ba6124b6830098 into 704f14f771b44af194e253bcc7b53af1(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:01,257 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:16:01,257 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/A, priority=13, startTime=1733814961192; duration=0sec 2024-12-10T07:16:01,257 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:16:01,257 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:A 2024-12-10T07:16:01,257 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:16:01,259 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:16:01,259 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/C is initiating minor compaction (all files) 2024-12-10T07:16:01,260 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/C in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:16:01,260 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/27b8e7a6c3e943aaa38c524601cd5e71, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/1a98c2201ce8465a981527a98b8dd6c4, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/a16ec531d54746a0a955a84e2a2a7bb1] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=37.0 K 2024-12-10T07:16:01,260 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27b8e7a6c3e943aaa38c524601cd5e71, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1733814958584 2024-12-10T07:16:01,260 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a98c2201ce8465a981527a98b8dd6c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1733814959230 2024-12-10T07:16:01,261 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting a16ec531d54746a0a955a84e2a2a7bb1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733814959898 2024-12-10T07:16:01,269 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#C#compaction#296 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:01,270 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/258656df764943bc9488e368022bb208 is 50, key is test_row_0/C:col10/1733814961045/Put/seqid=0 2024-12-10T07:16:01,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815021264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:01,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:01,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815021278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:01,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:01,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815021278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:01,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:01,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742171_1347 (size=13357) 2024-12-10T07:16:01,309 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/258656df764943bc9488e368022bb208 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/258656df764943bc9488e368022bb208 2024-12-10T07:16:01,317 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/C of 07d0f6bb02ffe54509ba6124b6830098 into 258656df764943bc9488e368022bb208(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:01,317 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:16:01,317 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/C, priority=13, startTime=1733814961209; duration=0sec 2024-12-10T07:16:01,317 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:01,317 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:C 2024-12-10T07:16:01,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815021384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:01,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815021395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:01,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815021395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-10T07:16:01,444 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:16:01,445 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:16:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-10T07:16:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T07:16:01,447 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:16:01,447 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:16:01,447 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:16:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T07:16:01,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815021595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:01,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-10T07:16:01,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:16:01,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:16:01,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:16:01,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:01,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:01,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815021604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:01,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:01,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815021604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:01,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:01,628 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/3102ceb6a6864aea9ac74f5f99ceca4f as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/3102ceb6a6864aea9ac74f5f99ceca4f 2024-12-10T07:16:01,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/7ab7cc2bf9084c19bf7e7d615c67ac5e 2024-12-10T07:16:01,632 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/B of 07d0f6bb02ffe54509ba6124b6830098 into 3102ceb6a6864aea9ac74f5f99ceca4f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:01,632 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:16:01,632 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/B, priority=13, startTime=1733814961195; duration=0sec 2024-12-10T07:16:01,632 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:01,633 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:B 2024-12-10T07:16:01,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/28c00e9336994c9c8a8e96909da7392b is 50, key is test_row_0/B:col10/1733814961208/Put/seqid=0 2024-12-10T07:16:01,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53046 deadline: 1733815021634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:01,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53098 deadline: 1733815021635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:01,637 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 07d0f6bb02ffe54509ba6124b6830098 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:01,638 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 07d0f6bb02ffe54509ba6124b6830098 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:01,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:01,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742172_1348 (size=12301) 2024-12-10T07:16:01,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-10T07:16:01,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:16:01,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:16:01,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:16:01,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:01,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T07:16:01,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-10T07:16:01,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:16:01,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:16:01,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:16:01,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:01,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:01,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:01,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815021905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:01,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:01,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815021915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815021915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:01,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:01,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:02,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/28c00e9336994c9c8a8e96909da7392b 2024-12-10T07:16:02,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/5a4f471c3f2e4e2592ffb4c9f44217ea is 50, key is test_row_0/C:col10/1733814961208/Put/seqid=0 2024-12-10T07:16:02,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742173_1349 (size=12301) 2024-12-10T07:16:02,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-10T07:16:02,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:16:02,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:16:02,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:16:02,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:02,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T07:16:02,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-10T07:16:02,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:16:02,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:16:02,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:16:02,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:02,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:02,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:02,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-10T07:16:02,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:16:02,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:16:02,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:16:02,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:02,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:02,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:02,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:02,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815022415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:02,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:02,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815022425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:02,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:02,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:02,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:02,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:02,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815022425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:02,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:02,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:02,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:02,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/5a4f471c3f2e4e2592ffb4c9f44217ea 2024-12-10T07:16:02,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/7ab7cc2bf9084c19bf7e7d615c67ac5e as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/7ab7cc2bf9084c19bf7e7d615c67ac5e 2024-12-10T07:16:02,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/7ab7cc2bf9084c19bf7e7d615c67ac5e, entries=250, sequenceid=502, filesize=16.8 K 2024-12-10T07:16:02,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/28c00e9336994c9c8a8e96909da7392b as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/28c00e9336994c9c8a8e96909da7392b 2024-12-10T07:16:02,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/28c00e9336994c9c8a8e96909da7392b, entries=150, sequenceid=502, filesize=12.0 K 2024-12-10T07:16:02,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/5a4f471c3f2e4e2592ffb4c9f44217ea as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/5a4f471c3f2e4e2592ffb4c9f44217ea 2024-12-10T07:16:02,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/5a4f471c3f2e4e2592ffb4c9f44217ea, entries=150, sequenceid=502, filesize=12.0 K 2024-12-10T07:16:02,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~91.69 KB/93890, heapSize ~267.14 KB/273552, currentSize=93.93 KB/96180 for 07d0f6bb02ffe54509ba6124b6830098 in 1263ms, sequenceid=502, compaction requested=false 2024-12-10T07:16:02,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:16:02,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-10T07:16:02,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:16:02,515 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T07:16:02,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:16:02,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:02,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:16:02,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:02,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:16:02,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:02,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/dd48ec2e73ba46b1adb64935f6e9ed8a is 50, key is test_row_0/A:col10/1733814961271/Put/seqid=0 2024-12-10T07:16:02,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742174_1350 (size=12301) 2024-12-10T07:16:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T07:16:02,928 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/dd48ec2e73ba46b1adb64935f6e9ed8a 2024-12-10T07:16:02,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/aa5d0422a32b40aa9dbac2f0df9a4131 is 50, key is test_row_0/B:col10/1733814961271/Put/seqid=0 2024-12-10T07:16:02,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742175_1351 (size=12301) 2024-12-10T07:16:03,340 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/aa5d0422a32b40aa9dbac2f0df9a4131 2024-12-10T07:16:03,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/9ad064b6a7b748f1a42ab21be1e7b62b is 50, key is test_row_0/C:col10/1733814961271/Put/seqid=0 2024-12-10T07:16:03,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742176_1352 (size=12301) 2024-12-10T07:16:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:16:03,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. as already flushing 2024-12-10T07:16:03,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815023449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:03,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815023449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:03,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:03,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:03,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:03,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:03,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:03,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:03,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815023450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:03,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:03,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:03,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:03,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:03,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815023554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:03,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815023565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:03,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:03,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:03,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:03,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815023565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:03,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:03,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:03,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T07:16:03,710 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T07:16:03,754 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/9ad064b6a7b748f1a42ab21be1e7b62b 2024-12-10T07:16:03,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/dd48ec2e73ba46b1adb64935f6e9ed8a as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/dd48ec2e73ba46b1adb64935f6e9ed8a 2024-12-10T07:16:03,762 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/dd48ec2e73ba46b1adb64935f6e9ed8a, entries=150, sequenceid=522, filesize=12.0 K 2024-12-10T07:16:03,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/aa5d0422a32b40aa9dbac2f0df9a4131 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/aa5d0422a32b40aa9dbac2f0df9a4131 2024-12-10T07:16:03,767 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/aa5d0422a32b40aa9dbac2f0df9a4131, entries=150, sequenceid=522, filesize=12.0 K 2024-12-10T07:16:03,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:03,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815023765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:03,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/9ad064b6a7b748f1a42ab21be1e7b62b as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/9ad064b6a7b748f1a42ab21be1e7b62b 2024-12-10T07:16:03,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:03,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:03,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:03,771 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/9ad064b6a7b748f1a42ab21be1e7b62b, entries=150, sequenceid=522, filesize=12.0 K 2024-12-10T07:16:03,772 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 07d0f6bb02ffe54509ba6124b6830098 in 1256ms, sequenceid=522, compaction requested=true 2024-12-10T07:16:03,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:16:03,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:16:03,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-10T07:16:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-10T07:16:03,774 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-10T07:16:03,774 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3260 sec 2024-12-10T07:16:03,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.3300 sec 2024-12-10T07:16:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:16:03,777 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T07:16:03,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:16:03,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:03,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:16:03,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:03,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:16:03,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:03,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/ce07347c83024a11ab10c2adbd1381aa is 50, key is test_row_0/A:col10/1733814963776/Put/seqid=0 2024-12-10T07:16:03,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742177_1353 (size=14741) 2024-12-10T07:16:03,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:03,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815023803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:03,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:03,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:03,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:03,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:03,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815023804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:03,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:03,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:03,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:03,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:03,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815023914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:03,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:03,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815023914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:03,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:03,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:03,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:03,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:03,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:03,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:04,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:04,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815024074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:04,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:04,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:04,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:04,086 INFO [Thread-1175 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1175. 2024-12-10T07:16:04,086 DEBUG [Thread-1175 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:16:04,086 DEBUG [Thread-1175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:04,086 DEBUG [Thread-1175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:04,086 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:16:04,086 INFO [Thread-1179 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1179. 2024-12-10T07:16:04,086 DEBUG [Thread-1179 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:16:04,086 DEBUG [Thread-1179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:04,086 INFO [Thread-1173 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1173. 2024-12-10T07:16:04,086 DEBUG [Thread-1179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:04,086 DEBUG [Thread-1173 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:16:04,086 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:16:04,086 DEBUG [Thread-1173 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:04,087 DEBUG [Thread-1173 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:04,087 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:16:04,090 INFO [Thread-1171 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1171. 2024-12-10T07:16:04,090 DEBUG [Thread-1171 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:16:04,090 DEBUG [Thread-1171 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:04,090 DEBUG [Thread-1171 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:04,090 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:16:04,090 INFO [Thread-1177 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1177. 2024-12-10T07:16:04,090 DEBUG [Thread-1177 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:16:04,090 DEBUG [Thread-1177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:04,090 DEBUG [Thread-1177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:04,090 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:16:04,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:04,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:04,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815024125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:04,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815024125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:04,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:04,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:04,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=538 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/ce07347c83024a11ab10c2adbd1381aa 2024-12-10T07:16:04,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/fdef7185e4e44e10a39be851ce74f860 is 50, key is test_row_0/B:col10/1733814963776/Put/seqid=0 2024-12-10T07:16:04,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742178_1354 (size=12301) 2024-12-10T07:16:04,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:04,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:04,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815024435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:04,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815024436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:04,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:04,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53126 deadline: 1733815024585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:04,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:04,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:04,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:04,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=538 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/fdef7185e4e44e10a39be851ce74f860 2024-12-10T07:16:04,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/878b6b41ae074d488b00cae9ee567ca3 is 50, key is test_row_0/C:col10/1733814963776/Put/seqid=0 2024-12-10T07:16:04,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742179_1355 (size=12301) 2024-12-10T07:16:04,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:04,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:04,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53110 deadline: 1733815024945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:04,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53074 deadline: 1733815024945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:04,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:04,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:04,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:04,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:04,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:04,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:05,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=538 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/878b6b41ae074d488b00cae9ee567ca3 2024-12-10T07:16:05,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/ce07347c83024a11ab10c2adbd1381aa as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/ce07347c83024a11ab10c2adbd1381aa 2024-12-10T07:16:05,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/ce07347c83024a11ab10c2adbd1381aa, entries=200, sequenceid=538, filesize=14.4 K 2024-12-10T07:16:05,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/fdef7185e4e44e10a39be851ce74f860 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/fdef7185e4e44e10a39be851ce74f860 2024-12-10T07:16:05,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/fdef7185e4e44e10a39be851ce74f860, entries=150, sequenceid=538, filesize=12.0 K 2024-12-10T07:16:05,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/878b6b41ae074d488b00cae9ee567ca3 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/878b6b41ae074d488b00cae9ee567ca3 2024-12-10T07:16:05,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/878b6b41ae074d488b00cae9ee567ca3, entries=150, sequenceid=538, filesize=12.0 K 2024-12-10T07:16:05,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 07d0f6bb02ffe54509ba6124b6830098 in 1244ms, sequenceid=538, compaction requested=true 2024-12-10T07:16:05,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:16:05,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:16:05,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:05,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:16:05,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:05,022 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:16:05,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07d0f6bb02ffe54509ba6124b6830098:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:16:05,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:16:05,022 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:16:05,023 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:16:05,023 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57580 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:16:05,023 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/B is initiating minor compaction (all files) 2024-12-10T07:16:05,023 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/A is initiating minor compaction (all files) 2024-12-10T07:16:05,023 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/B in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:16:05,023 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/A in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:16:05,023 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/3102ceb6a6864aea9ac74f5f99ceca4f, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/28c00e9336994c9c8a8e96909da7392b, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/aa5d0422a32b40aa9dbac2f0df9a4131, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/fdef7185e4e44e10a39be851ce74f860] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=49.1 K 2024-12-10T07:16:05,023 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/704f14f771b44af194e253bcc7b53af1, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/7ab7cc2bf9084c19bf7e7d615c67ac5e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/dd48ec2e73ba46b1adb64935f6e9ed8a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/ce07347c83024a11ab10c2adbd1381aa] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=56.2 K 2024-12-10T07:16:05,023 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 3102ceb6a6864aea9ac74f5f99ceca4f, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733814959898 2024-12-10T07:16:05,023 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 704f14f771b44af194e253bcc7b53af1, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733814959898 2024-12-10T07:16:05,023 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 28c00e9336994c9c8a8e96909da7392b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733814961070 2024-12-10T07:16:05,023 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ab7cc2bf9084c19bf7e7d615c67ac5e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733814961070 2024-12-10T07:16:05,024 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting aa5d0422a32b40aa9dbac2f0df9a4131, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1733814961260 2024-12-10T07:16:05,024 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd48ec2e73ba46b1adb64935f6e9ed8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1733814961260 2024-12-10T07:16:05,024 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting fdef7185e4e44e10a39be851ce74f860, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=538, earliestPutTs=1733814963445 2024-12-10T07:16:05,024 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce07347c83024a11ab10c2adbd1381aa, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=538, earliestPutTs=1733814963445 2024-12-10T07:16:05,030 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#A#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:05,030 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/501cbf91c49c4f0d98f462f5c1a99e42 is 50, key is test_row_0/A:col10/1733814963776/Put/seqid=0 2024-12-10T07:16:05,031 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#B#compaction#306 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:05,031 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/952f2ea2b065448c8d7ddcc62b37173e is 50, key is test_row_0/B:col10/1733814963776/Put/seqid=0 2024-12-10T07:16:05,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742180_1356 (size=13493) 2024-12-10T07:16:05,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742181_1357 (size=13493) 2024-12-10T07:16:05,442 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/501cbf91c49c4f0d98f462f5c1a99e42 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/501cbf91c49c4f0d98f462f5c1a99e42 2024-12-10T07:16:05,442 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/952f2ea2b065448c8d7ddcc62b37173e as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/952f2ea2b065448c8d7ddcc62b37173e 2024-12-10T07:16:05,447 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/B of 07d0f6bb02ffe54509ba6124b6830098 into 952f2ea2b065448c8d7ddcc62b37173e(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:05,447 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/A of 07d0f6bb02ffe54509ba6124b6830098 into 501cbf91c49c4f0d98f462f5c1a99e42(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:05,447 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:16:05,447 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:16:05,447 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/B, priority=12, startTime=1733814965022; duration=0sec 2024-12-10T07:16:05,447 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/A, priority=12, startTime=1733814965022; duration=0sec 2024-12-10T07:16:05,447 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:16:05,447 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:B 2024-12-10T07:16:05,447 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:16:05,447 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:A 2024-12-10T07:16:05,447 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:16:05,448 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:16:05,448 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 07d0f6bb02ffe54509ba6124b6830098/C is initiating minor compaction (all files) 2024-12-10T07:16:05,448 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07d0f6bb02ffe54509ba6124b6830098/C in TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:16:05,449 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/258656df764943bc9488e368022bb208, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/5a4f471c3f2e4e2592ffb4c9f44217ea, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/9ad064b6a7b748f1a42ab21be1e7b62b, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/878b6b41ae074d488b00cae9ee567ca3] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp, totalSize=49.1 K 2024-12-10T07:16:05,449 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 258656df764943bc9488e368022bb208, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733814959898 2024-12-10T07:16:05,449 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a4f471c3f2e4e2592ffb4c9f44217ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733814961070 2024-12-10T07:16:05,450 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ad064b6a7b748f1a42ab21be1e7b62b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1733814961260 2024-12-10T07:16:05,450 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 878b6b41ae074d488b00cae9ee567ca3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=538, earliestPutTs=1733814963445 2024-12-10T07:16:05,457 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07d0f6bb02ffe54509ba6124b6830098#C#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:05,458 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/2260f27045ce4d1593a5f8cf9a754621 is 50, key is test_row_0/C:col10/1733814963776/Put/seqid=0 2024-12-10T07:16:05,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742182_1358 (size=13493) 2024-12-10T07:16:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:16:05,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T07:16:05,600 INFO [Thread-1164 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1164. 2024-12-10T07:16:05,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:16:05,600 DEBUG [Thread-1164 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:16:05,600 DEBUG [Thread-1164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:05,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:05,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:16:05,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:05,600 DEBUG [Thread-1164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:05,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:16:05,600 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:16:05,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:05,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/c333477e8e2e4763b0e7ac3db9dc22a7 is 50, key is test_row_0/A:col10/1733814965596/Put/seqid=0 2024-12-10T07:16:05,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742183_1359 (size=12301) 2024-12-10T07:16:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-10T07:16:05,615 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:16:05,648 INFO [Thread-1166 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1166. 2024-12-10T07:16:05,648 INFO [Thread-1160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1160. 2024-12-10T07:16:05,648 DEBUG [Thread-1166 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:16:05,648 DEBUG [Thread-1160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:16:05,648 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:05,648 DEBUG [Thread-1160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:05,648 DEBUG [Thread-1160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:05,648 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:05,648 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:16:05,648 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:16:05,867 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/2260f27045ce4d1593a5f8cf9a754621 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/2260f27045ce4d1593a5f8cf9a754621 2024-12-10T07:16:05,874 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 07d0f6bb02ffe54509ba6124b6830098/C of 07d0f6bb02ffe54509ba6124b6830098 into 2260f27045ce4d1593a5f8cf9a754621(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:05,874 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:16:05,874 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., storeName=07d0f6bb02ffe54509ba6124b6830098/C, priority=12, startTime=1733814965022; duration=0sec 2024-12-10T07:16:05,874 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:05,874 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07d0f6bb02ffe54509ba6124b6830098:C 2024-12-10T07:16:05,956 INFO [Thread-1168 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1168. 2024-12-10T07:16:05,956 INFO [Thread-1162 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1162. 2024-12-10T07:16:05,956 DEBUG [Thread-1168 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:16:05,956 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:05,956 DEBUG [Thread-1162 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:16:05,956 DEBUG [Thread-1162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:05,956 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:05,956 DEBUG [Thread-1162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:05,956 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:16:05,956 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:16:05,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T07:16:05,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-12-10T07:16:05,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-12-10T07:16:05,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-12-10T07:16:05,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-12-10T07:16:05,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 138 2024-12-10T07:16:05,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T07:16:05,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2637 2024-12-10T07:16:05,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2714 2024-12-10T07:16:05,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2803 2024-12-10T07:16:05,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2651 2024-12-10T07:16:05,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2718 2024-12-10T07:16:05,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T07:16:05,957 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T07:16:05,957 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T07:16:05,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:05,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:05,957 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:16:05,958 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-10T07:16:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T07:16:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-10T07:16:05,961 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733814965961"}]},"ts":"1733814965961"} 2024-12-10T07:16:05,962 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T07:16:05,962 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T07:16:05,963 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T07:16:05,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=07d0f6bb02ffe54509ba6124b6830098, UNASSIGN}] 2024-12-10T07:16:05,965 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=07d0f6bb02ffe54509ba6124b6830098, UNASSIGN 2024-12-10T07:16:05,966 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=07d0f6bb02ffe54509ba6124b6830098, regionState=CLOSING, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:05,967 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=07d0f6bb02ffe54509ba6124b6830098, UNASSIGN because future has completed 2024-12-10T07:16:05,967 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T07:16:05,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835}] 2024-12-10T07:16:06,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/c333477e8e2e4763b0e7ac3db9dc22a7 2024-12-10T07:16:06,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/39ab621f8e8548de9caa787ce873837c is 50, key is test_row_0/B:col10/1733814965596/Put/seqid=0 2024-12-10T07:16:06,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742184_1360 (size=12301) 2024-12-10T07:16:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-10T07:16:06,118 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(122): Close 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:16:06,118 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T07:16:06,118 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1722): Closing 07d0f6bb02ffe54509ba6124b6830098, disabling compactions & flushes 2024-12-10T07:16:06,119 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:16:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-10T07:16:06,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/39ab621f8e8548de9caa787ce873837c 2024-12-10T07:16:06,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/a4657476bd614a089702276c7f171fc7 is 50, key is test_row_0/C:col10/1733814965596/Put/seqid=0 2024-12-10T07:16:06,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742185_1361 (size=12301) 2024-12-10T07:16:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-10T07:16:06,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/a4657476bd614a089702276c7f171fc7 2024-12-10T07:16:06,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/c333477e8e2e4763b0e7ac3db9dc22a7 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/c333477e8e2e4763b0e7ac3db9dc22a7 2024-12-10T07:16:06,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/c333477e8e2e4763b0e7ac3db9dc22a7, entries=150, sequenceid=557, filesize=12.0 K 2024-12-10T07:16:06,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/39ab621f8e8548de9caa787ce873837c as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/39ab621f8e8548de9caa787ce873837c 2024-12-10T07:16:06,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/39ab621f8e8548de9caa787ce873837c, entries=150, sequenceid=557, filesize=12.0 K 2024-12-10T07:16:06,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/a4657476bd614a089702276c7f171fc7 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/a4657476bd614a089702276c7f171fc7 2024-12-10T07:16:06,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/a4657476bd614a089702276c7f171fc7, entries=150, sequenceid=557, filesize=12.0 K 2024-12-10T07:16:06,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=26.84 KB/27480 for 07d0f6bb02ffe54509ba6124b6830098 in 1252ms, sequenceid=557, compaction requested=false 2024-12-10T07:16:06,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07d0f6bb02ffe54509ba6124b6830098: 2024-12-10T07:16:06,851 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:16:06,851 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:16:06,851 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. after waiting 0 ms 2024-12-10T07:16:06,851 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:16:06,851 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(2902): Flushing 07d0f6bb02ffe54509ba6124b6830098 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-10T07:16:06,851 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=A 2024-12-10T07:16:06,851 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:06,851 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=B 2024-12-10T07:16:06,851 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:06,851 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 07d0f6bb02ffe54509ba6124b6830098, store=C 2024-12-10T07:16:06,851 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:06,854 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/263c1131a64743d4b20ae3f9755d0970 is 50, key is test_row_0/A:col10/1733814965955/Put/seqid=0 2024-12-10T07:16:06,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742186_1362 (size=9857) 2024-12-10T07:16:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-10T07:16:07,258 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/263c1131a64743d4b20ae3f9755d0970 2024-12-10T07:16:07,271 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/eeae822de755498ea020629643648eac is 50, key is test_row_0/B:col10/1733814965955/Put/seqid=0 2024-12-10T07:16:07,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742187_1363 (size=9857) 2024-12-10T07:16:07,676 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/eeae822de755498ea020629643648eac 2024-12-10T07:16:07,689 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/e4f5000cec734ce2817aba2ce41cbc81 is 50, key is test_row_0/C:col10/1733814965955/Put/seqid=0 2024-12-10T07:16:07,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742188_1364 (size=9857) 2024-12-10T07:16:08,094 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=565 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/e4f5000cec734ce2817aba2ce41cbc81 2024-12-10T07:16:08,100 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/A/263c1131a64743d4b20ae3f9755d0970 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/263c1131a64743d4b20ae3f9755d0970 2024-12-10T07:16:08,105 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/263c1131a64743d4b20ae3f9755d0970, entries=100, sequenceid=565, filesize=9.6 K 2024-12-10T07:16:08,105 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/B/eeae822de755498ea020629643648eac as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/eeae822de755498ea020629643648eac 2024-12-10T07:16:08,108 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/eeae822de755498ea020629643648eac, entries=100, sequenceid=565, filesize=9.6 K 2024-12-10T07:16:08,109 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/.tmp/C/e4f5000cec734ce2817aba2ce41cbc81 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/e4f5000cec734ce2817aba2ce41cbc81 2024-12-10T07:16:08,112 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/e4f5000cec734ce2817aba2ce41cbc81, entries=100, sequenceid=565, filesize=9.6 K 2024-12-10T07:16:08,112 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 07d0f6bb02ffe54509ba6124b6830098 in 1261ms, sequenceid=565, compaction requested=true 2024-12-10T07:16:08,113 DEBUG [StoreCloser-TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/3c33a1519f0a41dba7159a0ea163a602, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/3bfb20db59b442b9a1ff5f9597e84f56, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/1af07eb810e7405d8271a04e32fc8305, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/a575bd6dd3004618a7c353053158c424, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/a14cbf2cb90e43a88b1830532dafd799, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/315e396bdfe4468bae7161b9acab02b4, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/9096ae1774af4231a5ec0501e407f77e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/9b62d060936c4bcab9a33123f017ef74, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/dbcf974c100d4946af77d8a136193327, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/5626c716617b40f08f830ff561777947, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/21199358a9654b21b93efdca05dfc728, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/606fa2a0888b41ee962205276a9dffb4, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/436b21ca297a4955a5eb40061f4896ed, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/56610babc4eb4a2db6b9903de51d5a58, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/86f51c63f9cb4051af0f224b13784b0d, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/53738b0ba8c748bc9c6bc55d0f9e72c0, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/1b001dd018e24955b8ecbc7ea9eeacf0, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/b959d2b2f0cb4fbf94c6a1404a378733, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/a706d063dd0e4db2becb6bc982318c6c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/509f220b2d814b2cbdf8d12eed0970a0, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/862aed12aba94a619c3222c43c9d91cd, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/2d0384138f994e9caec0cdbeb3556e05, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/2bdccafd4d42469ca3ac55d9443215ae, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/969970a436cf4f22b49306df5d7bdd9c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/220b8f60f0fa4da0aab2ca15249e807e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/952e308b617a4fe981b175dd6fd6ec39, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/6a2d90e60765431faa1548240f6e3260, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/fdfea959063c416b8f2bf4391df21461, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/311b2356ecce46ee84961ce7c36d1a98, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/e44a276637cd4d19a030da4cd996aaa1, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/704f14f771b44af194e253bcc7b53af1, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/224e67963d404caea4e653a0609d00a9, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/7ab7cc2bf9084c19bf7e7d615c67ac5e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/dd48ec2e73ba46b1adb64935f6e9ed8a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/ce07347c83024a11ab10c2adbd1381aa] to archive 2024-12-10T07:16:08,114 DEBUG [StoreCloser-TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T07:16:08,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-10T07:16:08,116 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/3bfb20db59b442b9a1ff5f9597e84f56 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/3bfb20db59b442b9a1ff5f9597e84f56 2024-12-10T07:16:08,116 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/1af07eb810e7405d8271a04e32fc8305 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/1af07eb810e7405d8271a04e32fc8305 2024-12-10T07:16:08,117 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/a575bd6dd3004618a7c353053158c424 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/a575bd6dd3004618a7c353053158c424 2024-12-10T07:16:08,117 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/3c33a1519f0a41dba7159a0ea163a602 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/3c33a1519f0a41dba7159a0ea163a602 2024-12-10T07:16:08,117 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/315e396bdfe4468bae7161b9acab02b4 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/315e396bdfe4468bae7161b9acab02b4 2024-12-10T07:16:08,117 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/a14cbf2cb90e43a88b1830532dafd799 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/a14cbf2cb90e43a88b1830532dafd799 2024-12-10T07:16:08,117 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/9096ae1774af4231a5ec0501e407f77e to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/9096ae1774af4231a5ec0501e407f77e 2024-12-10T07:16:08,117 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/9b62d060936c4bcab9a33123f017ef74 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/9b62d060936c4bcab9a33123f017ef74 2024-12-10T07:16:08,118 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/dbcf974c100d4946af77d8a136193327 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/dbcf974c100d4946af77d8a136193327 2024-12-10T07:16:08,118 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/5626c716617b40f08f830ff561777947 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/5626c716617b40f08f830ff561777947 2024-12-10T07:16:08,118 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/21199358a9654b21b93efdca05dfc728 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/21199358a9654b21b93efdca05dfc728 2024-12-10T07:16:08,119 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/436b21ca297a4955a5eb40061f4896ed to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/436b21ca297a4955a5eb40061f4896ed 2024-12-10T07:16:08,119 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/56610babc4eb4a2db6b9903de51d5a58 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/56610babc4eb4a2db6b9903de51d5a58 2024-12-10T07:16:08,119 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/606fa2a0888b41ee962205276a9dffb4 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/606fa2a0888b41ee962205276a9dffb4 2024-12-10T07:16:08,119 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/86f51c63f9cb4051af0f224b13784b0d to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/86f51c63f9cb4051af0f224b13784b0d 2024-12-10T07:16:08,119 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/53738b0ba8c748bc9c6bc55d0f9e72c0 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/53738b0ba8c748bc9c6bc55d0f9e72c0 2024-12-10T07:16:08,120 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/1b001dd018e24955b8ecbc7ea9eeacf0 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/1b001dd018e24955b8ecbc7ea9eeacf0 2024-12-10T07:16:08,120 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/a706d063dd0e4db2becb6bc982318c6c to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/a706d063dd0e4db2becb6bc982318c6c 2024-12-10T07:16:08,120 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/509f220b2d814b2cbdf8d12eed0970a0 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/509f220b2d814b2cbdf8d12eed0970a0 2024-12-10T07:16:08,120 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/b959d2b2f0cb4fbf94c6a1404a378733 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/b959d2b2f0cb4fbf94c6a1404a378733 2024-12-10T07:16:08,120 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/862aed12aba94a619c3222c43c9d91cd to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/862aed12aba94a619c3222c43c9d91cd 2024-12-10T07:16:08,120 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/2bdccafd4d42469ca3ac55d9443215ae to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/2bdccafd4d42469ca3ac55d9443215ae 2024-12-10T07:16:08,121 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/2d0384138f994e9caec0cdbeb3556e05 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/2d0384138f994e9caec0cdbeb3556e05 2024-12-10T07:16:08,121 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/969970a436cf4f22b49306df5d7bdd9c to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/969970a436cf4f22b49306df5d7bdd9c 2024-12-10T07:16:08,122 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/220b8f60f0fa4da0aab2ca15249e807e to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/220b8f60f0fa4da0aab2ca15249e807e 2024-12-10T07:16:08,122 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/6a2d90e60765431faa1548240f6e3260 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/6a2d90e60765431faa1548240f6e3260 2024-12-10T07:16:08,122 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/fdfea959063c416b8f2bf4391df21461 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/fdfea959063c416b8f2bf4391df21461 2024-12-10T07:16:08,124 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/311b2356ecce46ee84961ce7c36d1a98 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/311b2356ecce46ee84961ce7c36d1a98 2024-12-10T07:16:08,124 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/952e308b617a4fe981b175dd6fd6ec39 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/952e308b617a4fe981b175dd6fd6ec39 2024-12-10T07:16:08,124 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/704f14f771b44af194e253bcc7b53af1 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/704f14f771b44af194e253bcc7b53af1 2024-12-10T07:16:08,124 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/e44a276637cd4d19a030da4cd996aaa1 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/e44a276637cd4d19a030da4cd996aaa1 2024-12-10T07:16:08,124 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/224e67963d404caea4e653a0609d00a9 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/224e67963d404caea4e653a0609d00a9 2024-12-10T07:16:08,125 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/ce07347c83024a11ab10c2adbd1381aa to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/ce07347c83024a11ab10c2adbd1381aa 2024-12-10T07:16:08,125 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/7ab7cc2bf9084c19bf7e7d615c67ac5e to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/7ab7cc2bf9084c19bf7e7d615c67ac5e 2024-12-10T07:16:08,125 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/dd48ec2e73ba46b1adb64935f6e9ed8a to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/dd48ec2e73ba46b1adb64935f6e9ed8a 2024-12-10T07:16:08,126 DEBUG [StoreCloser-TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/7b5057f16f164f42b5cf9710c670b9da, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/e4220420895c4d6887c8992b7087d820, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/35128a9493474764ad55075063713ef1, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/61887389fd204d409311cf3d4f4ffeb4, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/782c8f98a8df43248dc2fe922a04bb49, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/00b26a55a948407ab95f4ce493bf2c02, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/fd3c075af7dc484d87652c1dad982f68, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/ece38bfb6fbe490ba9e192e9f3479719, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/19e4634d0c514ee38e84887b070c6518, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/04286781c03c4389b124ade95741279a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/7b0ccb475674452b8c8376a9590965f7, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/3f0d1d13291d4f36aae87f77ee11b9f3, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/6a136468a02a47c092ab09cd5fe509ba, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/eba764909bb64f1b975dd51a6fcd301c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/3c4b716a77e943c79e0269e44af76148, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/ac6c25645fe04625bab6ece17dd1f6df, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/7bbbc2341c0249389ee7a4855ef888a5, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/796d1e6edcc346e5b303d4540aac3951, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/823a48b873ef42e692c16b11a0660e0a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/53b00c42579740888366955c0a883078, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/755b4a2f748745b78557fd6d3a76bc7e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/2d68cd45f6544b8cb45fea661d2d3268, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/acdade7993814c6f8dd23ab43cd6d909, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/5faca50c3b824eb1802ce9363ecde377, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/bc44fa3426054e30b6fbdc11735e931d, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/7e097cb9dc7c4ffda1cf7b1bba144096, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/acefc8453a4e4f3893efa81bb724b31c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/9bf62b6517314638b798a56af3b61d14, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/aab45fa065384ad3a070866d34ee8feb, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/e30d731ec456494d8c90f6a6f2140e3e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/3102ceb6a6864aea9ac74f5f99ceca4f, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/c5be785f46d349828812b5fa772938a7, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/28c00e9336994c9c8a8e96909da7392b, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/aa5d0422a32b40aa9dbac2f0df9a4131, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/fdef7185e4e44e10a39be851ce74f860] to archive 2024-12-10T07:16:08,127 DEBUG [StoreCloser-TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T07:16:08,129 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/61887389fd204d409311cf3d4f4ffeb4 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/61887389fd204d409311cf3d4f4ffeb4 2024-12-10T07:16:08,129 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/782c8f98a8df43248dc2fe922a04bb49 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/782c8f98a8df43248dc2fe922a04bb49 2024-12-10T07:16:08,129 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/35128a9493474764ad55075063713ef1 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/35128a9493474764ad55075063713ef1 2024-12-10T07:16:08,129 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/7b5057f16f164f42b5cf9710c670b9da to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/7b5057f16f164f42b5cf9710c670b9da 2024-12-10T07:16:08,129 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/fd3c075af7dc484d87652c1dad982f68 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/fd3c075af7dc484d87652c1dad982f68 2024-12-10T07:16:08,129 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/00b26a55a948407ab95f4ce493bf2c02 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/00b26a55a948407ab95f4ce493bf2c02 2024-12-10T07:16:08,129 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/e4220420895c4d6887c8992b7087d820 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/e4220420895c4d6887c8992b7087d820 2024-12-10T07:16:08,129 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/ece38bfb6fbe490ba9e192e9f3479719 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/ece38bfb6fbe490ba9e192e9f3479719 2024-12-10T07:16:08,130 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/7b0ccb475674452b8c8376a9590965f7 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/7b0ccb475674452b8c8376a9590965f7 2024-12-10T07:16:08,130 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/19e4634d0c514ee38e84887b070c6518 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/19e4634d0c514ee38e84887b070c6518 2024-12-10T07:16:08,131 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/3f0d1d13291d4f36aae87f77ee11b9f3 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/3f0d1d13291d4f36aae87f77ee11b9f3 2024-12-10T07:16:08,131 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/04286781c03c4389b124ade95741279a to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/04286781c03c4389b124ade95741279a 2024-12-10T07:16:08,131 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/6a136468a02a47c092ab09cd5fe509ba to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/6a136468a02a47c092ab09cd5fe509ba 2024-12-10T07:16:08,131 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/eba764909bb64f1b975dd51a6fcd301c to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/eba764909bb64f1b975dd51a6fcd301c 2024-12-10T07:16:08,131 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/3c4b716a77e943c79e0269e44af76148 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/3c4b716a77e943c79e0269e44af76148 2024-12-10T07:16:08,131 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/ac6c25645fe04625bab6ece17dd1f6df to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/ac6c25645fe04625bab6ece17dd1f6df 2024-12-10T07:16:08,132 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/7bbbc2341c0249389ee7a4855ef888a5 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/7bbbc2341c0249389ee7a4855ef888a5 2024-12-10T07:16:08,132 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/796d1e6edcc346e5b303d4540aac3951 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/796d1e6edcc346e5b303d4540aac3951 2024-12-10T07:16:08,132 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/2d68cd45f6544b8cb45fea661d2d3268 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/2d68cd45f6544b8cb45fea661d2d3268 2024-12-10T07:16:08,132 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/755b4a2f748745b78557fd6d3a76bc7e to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/755b4a2f748745b78557fd6d3a76bc7e 2024-12-10T07:16:08,132 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/823a48b873ef42e692c16b11a0660e0a to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/823a48b873ef42e692c16b11a0660e0a 2024-12-10T07:16:08,133 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/acdade7993814c6f8dd23ab43cd6d909 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/acdade7993814c6f8dd23ab43cd6d909 2024-12-10T07:16:08,133 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/53b00c42579740888366955c0a883078 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/53b00c42579740888366955c0a883078 2024-12-10T07:16:08,133 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/5faca50c3b824eb1802ce9363ecde377 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/5faca50c3b824eb1802ce9363ecde377 2024-12-10T07:16:08,133 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/bc44fa3426054e30b6fbdc11735e931d to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/bc44fa3426054e30b6fbdc11735e931d 2024-12-10T07:16:08,134 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/7e097cb9dc7c4ffda1cf7b1bba144096 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/7e097cb9dc7c4ffda1cf7b1bba144096 2024-12-10T07:16:08,134 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/acefc8453a4e4f3893efa81bb724b31c to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/acefc8453a4e4f3893efa81bb724b31c 2024-12-10T07:16:08,134 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/9bf62b6517314638b798a56af3b61d14 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/9bf62b6517314638b798a56af3b61d14 2024-12-10T07:16:08,134 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/e30d731ec456494d8c90f6a6f2140e3e to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/e30d731ec456494d8c90f6a6f2140e3e 2024-12-10T07:16:08,134 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/aab45fa065384ad3a070866d34ee8feb to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/aab45fa065384ad3a070866d34ee8feb 2024-12-10T07:16:08,134 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/3102ceb6a6864aea9ac74f5f99ceca4f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/3102ceb6a6864aea9ac74f5f99ceca4f 2024-12-10T07:16:08,134 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/c5be785f46d349828812b5fa772938a7 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/c5be785f46d349828812b5fa772938a7 2024-12-10T07:16:08,135 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/aa5d0422a32b40aa9dbac2f0df9a4131 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/aa5d0422a32b40aa9dbac2f0df9a4131 2024-12-10T07:16:08,135 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/28c00e9336994c9c8a8e96909da7392b to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/28c00e9336994c9c8a8e96909da7392b 2024-12-10T07:16:08,135 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/fdef7185e4e44e10a39be851ce74f860 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/fdef7185e4e44e10a39be851ce74f860 2024-12-10T07:16:08,136 DEBUG [StoreCloser-TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/8ba9353b1f714e3c8f2c1190eb35b13d, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/11432f19ddaa46219b595d6e9a0ecaff, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/19f18c53be5a44b4b4633190f1c05cab, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/e7620151c98f4ae4ad80b842a02280f7, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/605476f6eaf74f0793166c946f4cb466, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/2f5e29906ad8431a8722f04e2423438c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/4e866abe61aa4c5ab89e827a657882e5, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/b2ac7f71811a43999ec692186536f8b8, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/1792ab05143d4ca2b73a9c7764ff63bd, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/2f66fcd28dd24de9b9d5f13f8bb9551e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/7b2708d1138e434bada57861edcd9dbd, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/b8d4c955bb074787a412243663dcfd70, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/73d4454464aa43778c107034cfb6a279, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/05dbe89c336f45a0a43caec7abad853a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/ed32063d07c64e6e997bb5f54bd200b1, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/ae5a8aa24e194f2e99a58899c76c2ef2, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/24562cc497b84d05b8314e769fb7f7cf, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/e8b4731fa2434caa841e9c76f2ede618, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/0c2c994798384c33be14ca5fd1c5c576, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/ef2a33f78a454debb32ace57bd35184b, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/0b1cc90e6453442f88fc7e966e538261, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/355f36b697964ca5b233e2cd3508be89, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/59b0f0f645b9447fb0dd7f6b40a596f2, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/743faaa3207c43c885e6ef96ea402f14, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/73ca0abe3c3746e19272b9b1663a515a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/40eccb25fe344ca19aa454c3d31d4169, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/6cd333e0cd4f4555ac34fb09117207a5, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/27b8e7a6c3e943aaa38c524601cd5e71, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/daee1449ecf343f3a483c7c7069e2fe0, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/1a98c2201ce8465a981527a98b8dd6c4, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/258656df764943bc9488e368022bb208, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/a16ec531d54746a0a955a84e2a2a7bb1, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/5a4f471c3f2e4e2592ffb4c9f44217ea, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/9ad064b6a7b748f1a42ab21be1e7b62b, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/878b6b41ae074d488b00cae9ee567ca3] to archive 2024-12-10T07:16:08,137 DEBUG [StoreCloser-TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T07:16:08,138 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/11432f19ddaa46219b595d6e9a0ecaff to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/11432f19ddaa46219b595d6e9a0ecaff 2024-12-10T07:16:08,138 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/19f18c53be5a44b4b4633190f1c05cab to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/19f18c53be5a44b4b4633190f1c05cab 2024-12-10T07:16:08,138 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/8ba9353b1f714e3c8f2c1190eb35b13d to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/8ba9353b1f714e3c8f2c1190eb35b13d 2024-12-10T07:16:08,139 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/605476f6eaf74f0793166c946f4cb466 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/605476f6eaf74f0793166c946f4cb466 2024-12-10T07:16:08,139 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/4e866abe61aa4c5ab89e827a657882e5 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/4e866abe61aa4c5ab89e827a657882e5 2024-12-10T07:16:08,139 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/2f5e29906ad8431a8722f04e2423438c to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/2f5e29906ad8431a8722f04e2423438c 2024-12-10T07:16:08,139 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/e7620151c98f4ae4ad80b842a02280f7 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/e7620151c98f4ae4ad80b842a02280f7 2024-12-10T07:16:08,139 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/b2ac7f71811a43999ec692186536f8b8 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/b2ac7f71811a43999ec692186536f8b8 2024-12-10T07:16:08,140 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/7b2708d1138e434bada57861edcd9dbd to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/7b2708d1138e434bada57861edcd9dbd 2024-12-10T07:16:08,140 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/1792ab05143d4ca2b73a9c7764ff63bd to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/1792ab05143d4ca2b73a9c7764ff63bd 2024-12-10T07:16:08,140 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/2f66fcd28dd24de9b9d5f13f8bb9551e to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/2f66fcd28dd24de9b9d5f13f8bb9551e 2024-12-10T07:16:08,141 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/b8d4c955bb074787a412243663dcfd70 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/b8d4c955bb074787a412243663dcfd70 2024-12-10T07:16:08,141 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/73d4454464aa43778c107034cfb6a279 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/73d4454464aa43778c107034cfb6a279 2024-12-10T07:16:08,141 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/ed32063d07c64e6e997bb5f54bd200b1 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/ed32063d07c64e6e997bb5f54bd200b1 2024-12-10T07:16:08,141 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/05dbe89c336f45a0a43caec7abad853a to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/05dbe89c336f45a0a43caec7abad853a 2024-12-10T07:16:08,142 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/ae5a8aa24e194f2e99a58899c76c2ef2 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/ae5a8aa24e194f2e99a58899c76c2ef2 2024-12-10T07:16:08,143 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/24562cc497b84d05b8314e769fb7f7cf to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/24562cc497b84d05b8314e769fb7f7cf 2024-12-10T07:16:08,143 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/e8b4731fa2434caa841e9c76f2ede618 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/e8b4731fa2434caa841e9c76f2ede618 2024-12-10T07:16:08,144 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/0c2c994798384c33be14ca5fd1c5c576 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/0c2c994798384c33be14ca5fd1c5c576 2024-12-10T07:16:08,144 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/ef2a33f78a454debb32ace57bd35184b to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/ef2a33f78a454debb32ace57bd35184b 2024-12-10T07:16:08,144 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/0b1cc90e6453442f88fc7e966e538261 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/0b1cc90e6453442f88fc7e966e538261 2024-12-10T07:16:08,144 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/355f36b697964ca5b233e2cd3508be89 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/355f36b697964ca5b233e2cd3508be89 2024-12-10T07:16:08,144 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/743faaa3207c43c885e6ef96ea402f14 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/743faaa3207c43c885e6ef96ea402f14 2024-12-10T07:16:08,144 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/59b0f0f645b9447fb0dd7f6b40a596f2 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/59b0f0f645b9447fb0dd7f6b40a596f2 2024-12-10T07:16:08,145 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/73ca0abe3c3746e19272b9b1663a515a to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/73ca0abe3c3746e19272b9b1663a515a 2024-12-10T07:16:08,145 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/6cd333e0cd4f4555ac34fb09117207a5 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/6cd333e0cd4f4555ac34fb09117207a5 2024-12-10T07:16:08,145 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/40eccb25fe344ca19aa454c3d31d4169 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/40eccb25fe344ca19aa454c3d31d4169 2024-12-10T07:16:08,145 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/27b8e7a6c3e943aaa38c524601cd5e71 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/27b8e7a6c3e943aaa38c524601cd5e71 2024-12-10T07:16:08,145 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/daee1449ecf343f3a483c7c7069e2fe0 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/daee1449ecf343f3a483c7c7069e2fe0 2024-12-10T07:16:08,146 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/1a98c2201ce8465a981527a98b8dd6c4 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/1a98c2201ce8465a981527a98b8dd6c4 2024-12-10T07:16:08,146 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/a16ec531d54746a0a955a84e2a2a7bb1 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/a16ec531d54746a0a955a84e2a2a7bb1 2024-12-10T07:16:08,146 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/258656df764943bc9488e368022bb208 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/258656df764943bc9488e368022bb208 2024-12-10T07:16:08,146 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/5a4f471c3f2e4e2592ffb4c9f44217ea to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/5a4f471c3f2e4e2592ffb4c9f44217ea 2024-12-10T07:16:08,146 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/9ad064b6a7b748f1a42ab21be1e7b62b to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/9ad064b6a7b748f1a42ab21be1e7b62b 2024-12-10T07:16:08,146 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/878b6b41ae074d488b00cae9ee567ca3 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/878b6b41ae074d488b00cae9ee567ca3 2024-12-10T07:16:08,149 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/recovered.edits/568.seqid, newMaxSeqId=568, maxSeqId=1 2024-12-10T07:16:08,150 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. 2024-12-10T07:16:08,150 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1676): Region close journal for 07d0f6bb02ffe54509ba6124b6830098: Waiting for close lock at 1733814966118Running coprocessor pre-close hooks at 1733814966118Disabling compacts and flushes for region at 1733814966118Disabling writes for close at 1733814966851 (+733 ms)Obtaining lock to block concurrent updates at 1733814966851Preparing flush snapshotting stores in 07d0f6bb02ffe54509ba6124b6830098 at 1733814966851Finished memstore snapshotting TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733814966851Flushing stores of TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098. at 1733814966852 (+1 ms)Flushing 07d0f6bb02ffe54509ba6124b6830098/A: creating writer at 1733814966852Flushing 07d0f6bb02ffe54509ba6124b6830098/A: appending metadata at 1733814966853 (+1 ms)Flushing 07d0f6bb02ffe54509ba6124b6830098/A: closing flushed file at 1733814966854 (+1 ms)Flushing 07d0f6bb02ffe54509ba6124b6830098/B: creating writer at 1733814967267 (+413 ms)Flushing 07d0f6bb02ffe54509ba6124b6830098/B: appending metadata at 1733814967270 (+3 ms)Flushing 07d0f6bb02ffe54509ba6124b6830098/B: closing flushed file at 1733814967270Flushing 07d0f6bb02ffe54509ba6124b6830098/C: creating writer at 1733814967685 (+415 ms)Flushing 07d0f6bb02ffe54509ba6124b6830098/C: appending metadata at 1733814967688 (+3 ms)Flushing 07d0f6bb02ffe54509ba6124b6830098/C: closing flushed file at 1733814967688Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c1f59d9: reopening flushed file at 1733814968099 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@270c9c06: reopening flushed file at 1733814968105 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@80f67e5: reopening flushed file at 1733814968109 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 07d0f6bb02ffe54509ba6124b6830098 in 1261ms, sequenceid=565, compaction requested=true at 1733814968112 (+3 ms)Writing region close event to WAL at 1733814968147 (+35 ms)Running coprocessor post-close hooks at 1733814968150 (+3 ms)Closed at 1733814968150 2024-12-10T07:16:08,151 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(157): Closed 07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:16:08,152 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=07d0f6bb02ffe54509ba6124b6830098, regionState=CLOSED 2024-12-10T07:16:08,153 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 because future has completed 2024-12-10T07:16:08,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-10T07:16:08,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseRegionProcedure 07d0f6bb02ffe54509ba6124b6830098, server=b1396b8d2d12,35871,1733814875835 in 2.1860 sec 2024-12-10T07:16:08,156 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-12-10T07:16:08,156 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=07d0f6bb02ffe54509ba6124b6830098, UNASSIGN in 2.1910 sec 2024-12-10T07:16:08,157 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-10T07:16:08,158 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.1930 sec 2024-12-10T07:16:08,159 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733814968158"}]},"ts":"1733814968158"} 2024-12-10T07:16:08,159 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T07:16:08,159 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T07:16:08,161 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.2020 sec 2024-12-10T07:16:10,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-10T07:16:10,145 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:16:10,146 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-10T07:16:10,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:16:10,148 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:16:10,149 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=82, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:16:10,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-10T07:16:10,151 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:16:10,154 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A, FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B, FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C, FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/recovered.edits] 2024-12-10T07:16:10,158 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/501cbf91c49c4f0d98f462f5c1a99e42 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/501cbf91c49c4f0d98f462f5c1a99e42 2024-12-10T07:16:10,158 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/263c1131a64743d4b20ae3f9755d0970 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/263c1131a64743d4b20ae3f9755d0970 2024-12-10T07:16:10,158 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/c333477e8e2e4763b0e7ac3db9dc22a7 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/A/c333477e8e2e4763b0e7ac3db9dc22a7 2024-12-10T07:16:10,160 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/39ab621f8e8548de9caa787ce873837c to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/39ab621f8e8548de9caa787ce873837c 2024-12-10T07:16:10,160 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/eeae822de755498ea020629643648eac to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/eeae822de755498ea020629643648eac 2024-12-10T07:16:10,160 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/952f2ea2b065448c8d7ddcc62b37173e to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/B/952f2ea2b065448c8d7ddcc62b37173e 2024-12-10T07:16:10,162 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/2260f27045ce4d1593a5f8cf9a754621 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/2260f27045ce4d1593a5f8cf9a754621 2024-12-10T07:16:10,162 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/e4f5000cec734ce2817aba2ce41cbc81 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/e4f5000cec734ce2817aba2ce41cbc81 2024-12-10T07:16:10,163 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/a4657476bd614a089702276c7f171fc7 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/C/a4657476bd614a089702276c7f171fc7 2024-12-10T07:16:10,165 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/recovered.edits/568.seqid to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098/recovered.edits/568.seqid 2024-12-10T07:16:10,165 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/07d0f6bb02ffe54509ba6124b6830098 2024-12-10T07:16:10,165 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T07:16:10,167 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=82, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:16:10,169 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T07:16:10,170 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T07:16:10,171 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=82, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:16:10,171 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T07:16:10,172 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733814970171"}]},"ts":"9223372036854775807"} 2024-12-10T07:16:10,173 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T07:16:10,173 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 07d0f6bb02ffe54509ba6124b6830098, NAME => 'TestAcidGuarantees,,1733814942845.07d0f6bb02ffe54509ba6124b6830098.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T07:16:10,173 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T07:16:10,173 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733814970173"}]},"ts":"9223372036854775807"} 2024-12-10T07:16:10,174 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T07:16:10,175 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=82, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:16:10,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 29 msec 2024-12-10T07:16:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-10T07:16:10,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T07:16:10,255 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:16:10,263 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=258 (was 258), OpenFileDescriptor=459 (was 465), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=960 (was 957) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3473 (was 3998) 2024-12-10T07:16:10,274 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=258, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=960, ProcessCount=11, AvailableMemoryMB=3473 2024-12-10T07:16:10,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T07:16:10,275 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T07:16:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T07:16:10,277 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T07:16:10,277 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:10,277 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 83 2024-12-10T07:16:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-10T07:16:10,277 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T07:16:10,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742189_1365 (size=1061) 2024-12-10T07:16:10,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-10T07:16:10,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-10T07:16:10,684 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8 2024-12-10T07:16:10,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742190_1366 (size=53) 2024-12-10T07:16:10,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-10T07:16:11,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T07:16:11,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6351062f9f9ac7f3c520d90dabe1dd24, disabling compactions & flushes 2024-12-10T07:16:11,090 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:11,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:11,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. after waiting 0 ms 2024-12-10T07:16:11,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:11,090 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:11,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6351062f9f9ac7f3c520d90dabe1dd24: Waiting for close lock at 1733814971090Disabling compacts and flushes for region at 1733814971090Disabling writes for close at 1733814971090Writing region close event to WAL at 1733814971090Closed at 1733814971090 2024-12-10T07:16:11,091 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T07:16:11,091 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733814971091"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733814971091"}]},"ts":"1733814971091"} 2024-12-10T07:16:11,093 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T07:16:11,094 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T07:16:11,094 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733814971094"}]},"ts":"1733814971094"} 2024-12-10T07:16:11,095 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T07:16:11,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6351062f9f9ac7f3c520d90dabe1dd24, ASSIGN}] 2024-12-10T07:16:11,096 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6351062f9f9ac7f3c520d90dabe1dd24, ASSIGN 2024-12-10T07:16:11,097 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6351062f9f9ac7f3c520d90dabe1dd24, ASSIGN; state=OFFLINE, location=b1396b8d2d12,35871,1733814875835; forceNewPlan=false, retain=false 2024-12-10T07:16:11,249 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=6351062f9f9ac7f3c520d90dabe1dd24, regionState=OPENING, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:11,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6351062f9f9ac7f3c520d90dabe1dd24, ASSIGN because future has completed 2024-12-10T07:16:11,254 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835}] 2024-12-10T07:16:11,412 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:11,412 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7752): Opening region: {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} 2024-12-10T07:16:11,412 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:11,412 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T07:16:11,412 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7794): checking encryption for 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:11,412 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7797): checking classloading for 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:11,413 INFO [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-10T07:16:11,414 INFO [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:16:11,414 INFO [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6351062f9f9ac7f3c520d90dabe1dd24 columnFamilyName A 2024-12-10T07:16:11,414 DEBUG [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:11,415 INFO [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] regionserver.HStore(327): Store=6351062f9f9ac7f3c520d90dabe1dd24/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:16:11,415 INFO [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:11,416 INFO [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:16:11,416 INFO [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6351062f9f9ac7f3c520d90dabe1dd24 columnFamilyName B 2024-12-10T07:16:11,416 DEBUG [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:11,416 INFO [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] regionserver.HStore(327): Store=6351062f9f9ac7f3c520d90dabe1dd24/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:16:11,416 INFO [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:11,417 INFO [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:16:11,417 INFO [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6351062f9f9ac7f3c520d90dabe1dd24 columnFamilyName C 2024-12-10T07:16:11,417 DEBUG [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:11,418 INFO [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] regionserver.HStore(327): Store=6351062f9f9ac7f3c520d90dabe1dd24/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:16:11,418 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:11,418 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1038): replaying wal for 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:11,418 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:11,419 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:11,419 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1048): stopping wal replay for 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:11,419 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1060): Cleaning up temporary data for 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:11,420 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T07:16:11,420 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1093): writing seq id for 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:11,422 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T07:16:11,422 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1114): Opened 6351062f9f9ac7f3c520d90dabe1dd24; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59357254, jitterRate=-0.11550799012184143}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T07:16:11,422 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:11,423 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1006): Region open journal for 6351062f9f9ac7f3c520d90dabe1dd24: Running coprocessor pre-open hook at 1733814971412Writing region info on filesystem at 1733814971412Initializing all the Stores at 1733814971413 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733814971413Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733814971413Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733814971413Cleaning up temporary data from old regions at 1733814971419 (+6 ms)Running coprocessor post-open hooks at 1733814971422 (+3 ms)Region opened successfully at 1733814971423 (+1 ms) 2024-12-10T07:16:11,423 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., pid=85, masterSystemTime=1733814971409 2024-12-10T07:16:11,425 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:11,425 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:11,425 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=6351062f9f9ac7f3c520d90dabe1dd24, regionState=OPEN, openSeqNum=2, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:11,427 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 because future has completed 2024-12-10T07:16:11,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-10T07:16:11,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure 6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 in 174 msec 2024-12-10T07:16:11,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-12-10T07:16:11,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6351062f9f9ac7f3c520d90dabe1dd24, ASSIGN in 333 msec 2024-12-10T07:16:11,430 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T07:16:11,431 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733814971430"}]},"ts":"1733814971430"} 2024-12-10T07:16:11,432 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T07:16:11,432 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T07:16:11,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1570 sec 2024-12-10T07:16:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-10T07:16:12,425 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:16:12,425 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff492b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:12,426 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:16:12,426 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:16:12,426 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:16:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:16:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:16:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a76c47e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:16:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:16:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:12,428 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:16:12,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23264ba4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:12,428 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:16:12,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:16:12,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:16:12,430 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:16:12,430 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T07:16:12,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is b1396b8d2d12,36259,1733814874775 2024-12-10T07:16:12,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@21deb5f 2024-12-10T07:16:12,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T07:16:12,432 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T07:16:12,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T07:16:12,433 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T07:16:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-10T07:16:12,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742191_1367 (size=1097) 2024-12-10T07:16:12,843 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-10T07:16:12,843 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-10T07:16:12,846 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T07:16:12,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6351062f9f9ac7f3c520d90dabe1dd24, REOPEN/MOVE}] 2024-12-10T07:16:12,849 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6351062f9f9ac7f3c520d90dabe1dd24, REOPEN/MOVE 2024-12-10T07:16:12,850 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=6351062f9f9ac7f3c520d90dabe1dd24, regionState=CLOSING, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:12,851 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6351062f9f9ac7f3c520d90dabe1dd24, REOPEN/MOVE because future has completed 2024-12-10T07:16:12,851 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T07:16:12,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835}] 2024-12-10T07:16:13,003 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:13,003 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T07:16:13,003 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing 6351062f9f9ac7f3c520d90dabe1dd24, disabling compactions & flushes 2024-12-10T07:16:13,003 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:13,003 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:13,003 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. after waiting 0 ms 2024-12-10T07:16:13,003 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:13,007 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-10T07:16:13,008 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:13,008 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for 6351062f9f9ac7f3c520d90dabe1dd24: Waiting for close lock at 1733814973003Running coprocessor pre-close hooks at 1733814973003Disabling compacts and flushes for region at 1733814973003Disabling writes for close at 1733814973003Writing region close event to WAL at 1733814973004 (+1 ms)Running coprocessor post-close hooks at 1733814973008 (+4 ms)Closed at 1733814973008 2024-12-10T07:16:13,008 WARN [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionServer(3299): Not adding moved region record: 6351062f9f9ac7f3c520d90dabe1dd24 to self. 2024-12-10T07:16:13,010 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:13,011 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=6351062f9f9ac7f3c520d90dabe1dd24, regionState=CLOSED 2024-12-10T07:16:13,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 because future has completed 2024-12-10T07:16:13,015 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-10T07:16:13,015 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure 6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 in 162 msec 2024-12-10T07:16:13,015 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6351062f9f9ac7f3c520d90dabe1dd24, REOPEN/MOVE; state=CLOSED, location=b1396b8d2d12,35871,1733814875835; forceNewPlan=false, retain=true 2024-12-10T07:16:13,166 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=6351062f9f9ac7f3c520d90dabe1dd24, regionState=OPENING, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:13,168 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6351062f9f9ac7f3c520d90dabe1dd24, REOPEN/MOVE because future has completed 2024-12-10T07:16:13,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835}] 2024-12-10T07:16:13,323 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:13,324 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7752): Opening region: {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} 2024-12-10T07:16:13,324 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:13,324 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T07:16:13,324 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7794): checking encryption for 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:13,324 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7797): checking classloading for 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:13,325 INFO [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:13,327 INFO [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:16:13,327 INFO [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6351062f9f9ac7f3c520d90dabe1dd24 columnFamilyName A 2024-12-10T07:16:13,328 DEBUG [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:13,328 INFO [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] regionserver.HStore(327): Store=6351062f9f9ac7f3c520d90dabe1dd24/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:16:13,329 INFO [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:13,329 INFO [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:16:13,330 INFO [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6351062f9f9ac7f3c520d90dabe1dd24 columnFamilyName B 2024-12-10T07:16:13,330 DEBUG [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:13,330 INFO [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] regionserver.HStore(327): Store=6351062f9f9ac7f3c520d90dabe1dd24/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:16:13,330 INFO [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:13,331 INFO [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:16:13,331 INFO [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6351062f9f9ac7f3c520d90dabe1dd24 columnFamilyName C 2024-12-10T07:16:13,331 DEBUG [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:13,331 INFO [StoreOpener-6351062f9f9ac7f3c520d90dabe1dd24-1 {}] regionserver.HStore(327): Store=6351062f9f9ac7f3c520d90dabe1dd24/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:16:13,332 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:13,332 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1038): replaying wal for 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:13,332 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:13,333 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:13,334 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1048): stopping wal replay for 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:13,334 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1060): Cleaning up temporary data for 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:13,335 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T07:16:13,336 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1093): writing seq id for 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:13,337 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1114): Opened 6351062f9f9ac7f3c520d90dabe1dd24; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60496832, jitterRate=-0.0985269546508789}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T07:16:13,337 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:13,338 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1006): Region open journal for 6351062f9f9ac7f3c520d90dabe1dd24: Running coprocessor pre-open hook at 1733814973324Writing region info on filesystem at 1733814973324Initializing all the Stores at 1733814973325 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733814973325Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733814973325Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733814973325Cleaning up temporary data from old regions at 1733814973334 (+9 ms)Running coprocessor post-open hooks at 1733814973337 (+3 ms)Region opened successfully at 1733814973338 (+1 ms) 2024-12-10T07:16:13,339 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., pid=90, masterSystemTime=1733814973321 2024-12-10T07:16:13,342 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:13,342 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:13,342 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=6351062f9f9ac7f3c520d90dabe1dd24, regionState=OPEN, openSeqNum=5, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:13,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 because future has completed 2024-12-10T07:16:13,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=88 2024-12-10T07:16:13,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 in 175 msec 2024-12-10T07:16:13,348 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-10T07:16:13,348 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6351062f9f9ac7f3c520d90dabe1dd24, REOPEN/MOVE in 498 msec 2024-12-10T07:16:13,351 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-10T07:16:13,351 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 504 msec 2024-12-10T07:16:13,354 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 919 msec 2024-12-10T07:16:13,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-10T07:16:13,355 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:16:13,356 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25e62965, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,356 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:16:13,356 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:16:13,357 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:16:13,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:16:13,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:16:13,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15d1bdb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:16:13,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:16:13,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:13,359 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:16:13,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5816fdda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,361 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e3f6460, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,361 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:16:13,361 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:16:13,361 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:16:13,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:16:13,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:16:13,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4746fffc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:16:13,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:16:13,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:13,363 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:16:13,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@311621ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,365 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@223c8c7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,365 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:16:13,365 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:16:13,366 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:16:13,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:16:13,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:16:13,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b0655, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:16:13,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:16:13,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:13,367 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:16:13,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15a41a01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,369 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b2eed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,369 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:16:13,369 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:16:13,370 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:16:13,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:16:13,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:16:13,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50fc5713, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:16:13,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:16:13,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:13,371 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:16:13,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@735b5e60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@469d248f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,373 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:16:13,373 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:16:13,374 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:16:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:16:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:16:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a395f3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:16:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:16:13,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:13,376 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:16:13,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c737991, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,377 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@120238c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,377 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:16:13,377 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:16:13,378 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:16:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:16:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:16:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2021860b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:16:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:16:13,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:13,382 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:16:13,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38e90f4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,384 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bdcd699, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,384 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:16:13,384 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:16:13,385 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:16:13,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:16:13,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:16:13,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a3cb0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:16:13,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:16:13,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:13,386 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:16:13,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46b14b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,389 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b20e1d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,389 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:16:13,389 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:16:13,390 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:16:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:16:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:16:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22ce678f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:16:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:16:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:13,392 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:16:13,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@547c689f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,393 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fbfe4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,393 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:16:13,393 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:16:13,394 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:16:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:16:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:16:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40c421a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:16:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:16:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:13,396 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:16:13,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10400d82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,398 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69a3307d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,398 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:16:13,398 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:16:13,399 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:16:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:16:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:16:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e7e2963, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:16:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:16:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:13,401 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:16:13,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@490959a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:13,405 DEBUG [Thread-1675 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:16:13,405 DEBUG [Thread-1658 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:16:13,405 DEBUG [Thread-1671 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:16:13,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:16:13,406 DEBUG [Thread-1671 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:16:13,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:16:13,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:16:13,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:16:13,407 DEBUG [Thread-1656 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:16:13,409 DEBUG [Thread-1667 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:16:13,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:16:13,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:16:13,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:16:13,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:16:13,410 DEBUG [Thread-1671 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:16:13,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:16:13,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:16:13,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:16:13,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:16:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:16:13,417 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:16:13,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:16:13,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:16:13,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:16:13,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:16:13,425 DEBUG [Thread-1669 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:16:13,425 DEBUG [Thread-1660 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:16:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:16:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:16:13,429 DEBUG [Thread-1664 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:16:13,429 DEBUG [Thread-1662 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:16:13,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:16:13,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:16:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:16:13,431 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:16:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:16:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:16:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:16:13,431 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:16:13,433 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:16:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:16:13,434 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:16:13,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:16:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:16:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:16:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:13,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6351062f9f9ac7f3c520d90dabe1dd24 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T07:16:13,442 DEBUG [Thread-1673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:16:13,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:16:13,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:16:13,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=A 2024-12-10T07:16:13,444 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:16:13,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:13,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=B 2024-12-10T07:16:13,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:13,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=C 2024-12-10T07:16:13,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:13,445 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:16:13,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:16:13,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-12-10T07:16:13,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-10T07:16:13,450 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:16:13,452 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:16:13,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:16:13,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121071004febae4743ec9b960d7cfea9edfc_6351062f9f9ac7f3c520d90dabe1dd24 is 50, key is test_row_0/A:col10/1733814973439/Put/seqid=0 2024-12-10T07:16:13,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:13,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48852 deadline: 1733815033496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:13,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:13,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815033503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:13,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:13,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48868 deadline: 1733815033507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:13,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815033508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:13,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48848 deadline: 1733815033508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:13,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742192_1368 (size=12154) 2024-12-10T07:16:13,551 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:13,558 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121071004febae4743ec9b960d7cfea9edfc_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121071004febae4743ec9b960d7cfea9edfc_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:13,559 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/8e7f2e03ca7747caa15ed01ce321c6e7, store: [table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:13,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/8e7f2e03ca7747caa15ed01ce321c6e7 is 175, key is test_row_0/A:col10/1733814973439/Put/seqid=0 2024-12-10T07:16:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-10T07:16:13,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742193_1369 (size=30955) 2024-12-10T07:16:13,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T07:16:13,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:13,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:13,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:13,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:13,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:13,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:13,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:13,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48852 deadline: 1733815033615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:13,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:13,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815033624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:13,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:13,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:13,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815033624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:13,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:13,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:13,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48848 deadline: 1733815033625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:13,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:13,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:13,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48868 deadline: 1733815033625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:13,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:13,673 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T07:16:13,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T07:16:13,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:13,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:13,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:13,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:13,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:13,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:13,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-10T07:16:13,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:13,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48852 deadline: 1733815033824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:13,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:13,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815033835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:13,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48848 deadline: 1733815033835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:13,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:13,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48868 deadline: 1733815033835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:13,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815033835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:13,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T07:16:13,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:13,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:13,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:13,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:13,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:13,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:13,993 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/8e7f2e03ca7747caa15ed01ce321c6e7 2024-12-10T07:16:14,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/0d1a2c5696944b20a2983817099eaff6 is 50, key is test_row_0/B:col10/1733814973439/Put/seqid=0 2024-12-10T07:16:14,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742194_1370 (size=12001) 2024-12-10T07:16:14,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/0d1a2c5696944b20a2983817099eaff6 2024-12-10T07:16:14,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T07:16:14,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:14,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:14,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:14,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:14,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:14,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:14,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/2e9704620a554ed48e96bbac8755f9ba is 50, key is test_row_0/C:col10/1733814973439/Put/seqid=0 2024-12-10T07:16:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-10T07:16:14,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742195_1371 (size=12001) 2024-12-10T07:16:14,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/2e9704620a554ed48e96bbac8755f9ba 2024-12-10T07:16:14,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/8e7f2e03ca7747caa15ed01ce321c6e7 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/8e7f2e03ca7747caa15ed01ce321c6e7 2024-12-10T07:16:14,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/8e7f2e03ca7747caa15ed01ce321c6e7, entries=150, sequenceid=17, filesize=30.2 K 2024-12-10T07:16:14,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/0d1a2c5696944b20a2983817099eaff6 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/0d1a2c5696944b20a2983817099eaff6 2024-12-10T07:16:14,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48852 deadline: 1733815034134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:14,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:14,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/0d1a2c5696944b20a2983817099eaff6, entries=150, sequenceid=17, filesize=11.7 K 2024-12-10T07:16:14,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:14,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48848 deadline: 1733815034149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:14,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:14,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/2e9704620a554ed48e96bbac8755f9ba as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/2e9704620a554ed48e96bbac8755f9ba 2024-12-10T07:16:14,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48868 deadline: 1733815034155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:14,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815034155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:14,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:14,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:14,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815034155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:14,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:14,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/2e9704620a554ed48e96bbac8755f9ba, entries=150, sequenceid=17, filesize=11.7 K 2024-12-10T07:16:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:14,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6351062f9f9ac7f3c520d90dabe1dd24 in 725ms, sequenceid=17, compaction requested=false 2024-12-10T07:16:14,165 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-10T07:16:14,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:14,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-10T07:16:14,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:14,225 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 6351062f9f9ac7f3c520d90dabe1dd24 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T07:16:14,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=A 2024-12-10T07:16:14,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:14,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=B 2024-12-10T07:16:14,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:14,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=C 2024-12-10T07:16:14,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:14,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210fc40b51efa3e4f34a60d3d51f47508df_6351062f9f9ac7f3c520d90dabe1dd24 is 50, key is test_row_0/A:col10/1733814973506/Put/seqid=0 2024-12-10T07:16:14,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742196_1372 (size=12154) 2024-12-10T07:16:14,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-10T07:16:14,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:14,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:14,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:14,651 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210fc40b51efa3e4f34a60d3d51f47508df_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210fc40b51efa3e4f34a60d3d51f47508df_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:14,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/85a4b232bada44e694b37191a46759ff, store: [table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:14,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/85a4b232bada44e694b37191a46759ff is 175, key is test_row_0/A:col10/1733814973506/Put/seqid=0 2024-12-10T07:16:14,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742197_1373 (size=30955) 2024-12-10T07:16:14,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48852 deadline: 1733815034673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:14,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815034674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815034674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48848 deadline: 1733815034679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:14,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:14,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48868 deadline: 1733815034679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:14,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:14,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:14,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:14,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48852 deadline: 1733815034784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:14,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48848 deadline: 1733815034784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48868 deadline: 1733815034784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:14,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48852 deadline: 1733815034995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:14,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48848 deadline: 1733815034995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:14,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:14,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:14,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48868 deadline: 1733815034995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:14,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:15,056 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/85a4b232bada44e694b37191a46759ff 2024-12-10T07:16:15,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/5d642eaf950444b58ddd9c6b0151ade1 is 50, key is test_row_0/B:col10/1733814973506/Put/seqid=0 2024-12-10T07:16:15,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742198_1374 (size=12001) 2024-12-10T07:16:15,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:15,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48852 deadline: 1733815035305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:15,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:15,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:15,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:15,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:15,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48848 deadline: 1733815035305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:15,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:15,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48868 deadline: 1733815035305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:15,468 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/5d642eaf950444b58ddd9c6b0151ade1 2024-12-10T07:16:15,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/50017646c67f44e7ae6ae34e655ae7bd is 50, key is test_row_0/C:col10/1733814973506/Put/seqid=0 2024-12-10T07:16:15,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742199_1375 (size=12001) 2024-12-10T07:16:15,499 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/50017646c67f44e7ae6ae34e655ae7bd 2024-12-10T07:16:15,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/85a4b232bada44e694b37191a46759ff as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/85a4b232bada44e694b37191a46759ff 2024-12-10T07:16:15,522 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/85a4b232bada44e694b37191a46759ff, entries=150, sequenceid=36, filesize=30.2 K 2024-12-10T07:16:15,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/5d642eaf950444b58ddd9c6b0151ade1 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/5d642eaf950444b58ddd9c6b0151ade1 2024-12-10T07:16:15,554 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/5d642eaf950444b58ddd9c6b0151ade1, entries=150, sequenceid=36, filesize=11.7 K 2024-12-10T07:16:15,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/50017646c67f44e7ae6ae34e655ae7bd as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/50017646c67f44e7ae6ae34e655ae7bd 2024-12-10T07:16:15,560 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/50017646c67f44e7ae6ae34e655ae7bd, entries=150, sequenceid=36, filesize=11.7 K 2024-12-10T07:16:15,561 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6351062f9f9ac7f3c520d90dabe1dd24 in 1336ms, sequenceid=36, compaction requested=false 2024-12-10T07:16:15,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:15,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:15,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-12-10T07:16:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-12-10T07:16:15,565 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-10T07:16:15,565 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1110 sec 2024-12-10T07:16:15,567 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 2.1210 sec 2024-12-10T07:16:15,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-10T07:16:15,633 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:16:15,633 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:16:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-10T07:16:15,637 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:16:15,639 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:16:15,639 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:16:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-10T07:16:15,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:15,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6351062f9f9ac7f3c520d90dabe1dd24 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T07:16:15,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=A 2024-12-10T07:16:15,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:15,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=B 2024-12-10T07:16:15,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:15,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=C 2024-12-10T07:16:15,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:15,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412101c9df304aa104ca8a13a49110b2b14dc_6351062f9f9ac7f3c520d90dabe1dd24 is 50, key is test_row_0/A:col10/1733814975698/Put/seqid=0 2024-12-10T07:16:15,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742200_1376 (size=12154) 2024-12-10T07:16:15,741 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:15,748 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412101c9df304aa104ca8a13a49110b2b14dc_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412101c9df304aa104ca8a13a49110b2b14dc_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:15,749 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/4ecfcf9aa5184aa9a0bffd4f06cd4d2d, store: [table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:15,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/4ecfcf9aa5184aa9a0bffd4f06cd4d2d is 175, key is test_row_0/A:col10/1733814975698/Put/seqid=0 2024-12-10T07:16:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-10T07:16:15,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742201_1377 (size=30955) 2024-12-10T07:16:15,771 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/4ecfcf9aa5184aa9a0bffd4f06cd4d2d 2024-12-10T07:16:15,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/c8722539973e4fc9a4de73c26283006e is 50, key is test_row_0/B:col10/1733814975698/Put/seqid=0 2024-12-10T07:16:15,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742202_1378 (size=12001) 2024-12-10T07:16:15,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:15,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815035779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:15,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/c8722539973e4fc9a4de73c26283006e 2024-12-10T07:16:15,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:15,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815035788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:15,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-10T07:16:15,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:15,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:15,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:15,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:15,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:15,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:15,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/d82b17b294ed4f5d81b80c1ab7e5b15d is 50, key is test_row_0/C:col10/1733814975698/Put/seqid=0 2024-12-10T07:16:15,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742203_1379 (size=12001) 2024-12-10T07:16:15,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:15,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48852 deadline: 1733815035825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:15,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:15,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48848 deadline: 1733815035825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:15,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:15,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:15,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:15,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:15,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:15,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48868 deadline: 1733815035825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:15,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:15,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:15,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:15,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:15,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:15,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815035895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:15,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:15,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815035895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:15,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:15,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:15,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:15,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-10T07:16:15,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:15,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:15,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:15,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:15,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:15,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:15,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-10T07:16:16,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-10T07:16:16,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:16,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:16,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:16,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:16,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:16,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:16,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:16,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815036104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:16,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:16,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815036105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:16,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/d82b17b294ed4f5d81b80c1ab7e5b15d 2024-12-10T07:16:16,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/4ecfcf9aa5184aa9a0bffd4f06cd4d2d as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/4ecfcf9aa5184aa9a0bffd4f06cd4d2d 2024-12-10T07:16:16,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/4ecfcf9aa5184aa9a0bffd4f06cd4d2d, entries=150, sequenceid=50, filesize=30.2 K 2024-12-10T07:16:16,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/c8722539973e4fc9a4de73c26283006e as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/c8722539973e4fc9a4de73c26283006e 2024-12-10T07:16:16,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/c8722539973e4fc9a4de73c26283006e, entries=150, sequenceid=50, filesize=11.7 K 2024-12-10T07:16:16,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/d82b17b294ed4f5d81b80c1ab7e5b15d as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/d82b17b294ed4f5d81b80c1ab7e5b15d 2024-12-10T07:16:16,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/d82b17b294ed4f5d81b80c1ab7e5b15d, entries=150, sequenceid=50, filesize=11.7 K 2024-12-10T07:16:16,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6351062f9f9ac7f3c520d90dabe1dd24 in 524ms, sequenceid=50, compaction requested=true 2024-12-10T07:16:16,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:16,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6351062f9f9ac7f3c520d90dabe1dd24:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:16:16,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:16,223 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:16:16,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6351062f9f9ac7f3c520d90dabe1dd24:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:16:16,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:16,223 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:16:16,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6351062f9f9ac7f3c520d90dabe1dd24:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:16:16,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:16:16,224 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:16:16,224 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:16:16,225 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 6351062f9f9ac7f3c520d90dabe1dd24/A is initiating minor compaction (all files) 2024-12-10T07:16:16,225 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 6351062f9f9ac7f3c520d90dabe1dd24/B is initiating minor compaction (all files) 2024-12-10T07:16:16,225 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6351062f9f9ac7f3c520d90dabe1dd24/B in TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:16,225 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6351062f9f9ac7f3c520d90dabe1dd24/A in TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:16,225 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/8e7f2e03ca7747caa15ed01ce321c6e7, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/85a4b232bada44e694b37191a46759ff, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/4ecfcf9aa5184aa9a0bffd4f06cd4d2d] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp, totalSize=90.7 K 2024-12-10T07:16:16,225 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/0d1a2c5696944b20a2983817099eaff6, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/5d642eaf950444b58ddd9c6b0151ade1, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/c8722539973e4fc9a4de73c26283006e] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp, totalSize=35.2 K 2024-12-10T07:16:16,225 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:16,225 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. files: [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/8e7f2e03ca7747caa15ed01ce321c6e7, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/85a4b232bada44e694b37191a46759ff, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/4ecfcf9aa5184aa9a0bffd4f06cd4d2d] 2024-12-10T07:16:16,225 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e7f2e03ca7747caa15ed01ce321c6e7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733814973437 2024-12-10T07:16:16,225 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d1a2c5696944b20a2983817099eaff6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733814973437 2024-12-10T07:16:16,225 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d642eaf950444b58ddd9c6b0151ade1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733814973505 2024-12-10T07:16:16,225 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85a4b232bada44e694b37191a46759ff, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733814973505 2024-12-10T07:16:16,226 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting c8722539973e4fc9a4de73c26283006e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733814974670 2024-12-10T07:16:16,226 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ecfcf9aa5184aa9a0bffd4f06cd4d2d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733814974670 2024-12-10T07:16:16,247 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6351062f9f9ac7f3c520d90dabe1dd24#B#compaction#323 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:16,248 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/1afbe93976f947da8cbe8fdcb5aca701 is 50, key is test_row_0/B:col10/1733814975698/Put/seqid=0 2024-12-10T07:16:16,249 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:16,252 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412107b68d59d95494e65b0947cab0a47d25b_6351062f9f9ac7f3c520d90dabe1dd24 store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:16,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-10T07:16:16,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:16,253 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 6351062f9f9ac7f3c520d90dabe1dd24 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T07:16:16,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=A 2024-12-10T07:16:16,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:16,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=B 2024-12-10T07:16:16,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:16,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=C 2024-12-10T07:16:16,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:16,254 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412107b68d59d95494e65b0947cab0a47d25b_6351062f9f9ac7f3c520d90dabe1dd24, store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:16,254 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412107b68d59d95494e65b0947cab0a47d25b_6351062f9f9ac7f3c520d90dabe1dd24 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:16,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742204_1380 (size=12104) 2024-12-10T07:16:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-10T07:16:16,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f6ee4fe1a5b94060be9bb0cde9869a80_6351062f9f9ac7f3c520d90dabe1dd24 is 50, key is test_row_0/A:col10/1733814975786/Put/seqid=0 2024-12-10T07:16:16,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742205_1381 (size=4469) 2024-12-10T07:16:16,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742206_1382 (size=12154) 2024-12-10T07:16:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:16,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:16,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815036449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:16,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815036455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:16,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:16,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:16,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:16,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:16,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:16,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:16,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815036565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:16,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:16,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815036565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:16,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:16,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:16,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:16,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:16,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:16,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:16,667 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/1afbe93976f947da8cbe8fdcb5aca701 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/1afbe93976f947da8cbe8fdcb5aca701 2024-12-10T07:16:16,671 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6351062f9f9ac7f3c520d90dabe1dd24/B of 6351062f9f9ac7f3c520d90dabe1dd24 into 1afbe93976f947da8cbe8fdcb5aca701(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:16,671 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:16,671 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., storeName=6351062f9f9ac7f3c520d90dabe1dd24/B, priority=13, startTime=1733814976223; duration=0sec 2024-12-10T07:16:16,671 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:16:16,671 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6351062f9f9ac7f3c520d90dabe1dd24:B 2024-12-10T07:16:16,671 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:16:16,686 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:16:16,687 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 6351062f9f9ac7f3c520d90dabe1dd24/C is initiating minor compaction (all files) 2024-12-10T07:16:16,687 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6351062f9f9ac7f3c520d90dabe1dd24/C in TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:16,687 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/2e9704620a554ed48e96bbac8755f9ba, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/50017646c67f44e7ae6ae34e655ae7bd, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/d82b17b294ed4f5d81b80c1ab7e5b15d] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp, totalSize=35.2 K 2024-12-10T07:16:16,687 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e9704620a554ed48e96bbac8755f9ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733814973437 2024-12-10T07:16:16,689 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 50017646c67f44e7ae6ae34e655ae7bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733814973505 2024-12-10T07:16:16,690 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting d82b17b294ed4f5d81b80c1ab7e5b15d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733814974670 2024-12-10T07:16:16,700 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6351062f9f9ac7f3c520d90dabe1dd24#A#compaction#324 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:16,701 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/58309db390694cd2901f6df07a2a5646 is 175, key is test_row_0/A:col10/1733814975698/Put/seqid=0 2024-12-10T07:16:16,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:16,711 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6351062f9f9ac7f3c520d90dabe1dd24#C#compaction#326 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:16,711 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/005f222920cf4d5da31284acd9af52a7 is 50, key is test_row_0/C:col10/1733814975698/Put/seqid=0 2024-12-10T07:16:16,711 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f6ee4fe1a5b94060be9bb0cde9869a80_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f6ee4fe1a5b94060be9bb0cde9869a80_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:16,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/63b861846f1a438f9ce1eba8cba99ab2, store: [table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:16,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/63b861846f1a438f9ce1eba8cba99ab2 is 175, key is test_row_0/A:col10/1733814975786/Put/seqid=0 2024-12-10T07:16:16,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742207_1383 (size=31058) 2024-12-10T07:16:16,732 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/58309db390694cd2901f6df07a2a5646 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/58309db390694cd2901f6df07a2a5646 2024-12-10T07:16:16,736 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6351062f9f9ac7f3c520d90dabe1dd24/A of 6351062f9f9ac7f3c520d90dabe1dd24 into 58309db390694cd2901f6df07a2a5646(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:16,736 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:16,736 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., storeName=6351062f9f9ac7f3c520d90dabe1dd24/A, priority=13, startTime=1733814976223; duration=0sec 2024-12-10T07:16:16,736 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:16,736 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6351062f9f9ac7f3c520d90dabe1dd24:A 2024-12-10T07:16:16,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742209_1385 (size=30955) 2024-12-10T07:16:16,759 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/63b861846f1a438f9ce1eba8cba99ab2 2024-12-10T07:16:16,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/100bc14414684003ab8acf377c2039e6 is 50, key is test_row_0/B:col10/1733814975786/Put/seqid=0 2024-12-10T07:16:16,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742208_1384 (size=12104) 2024-12-10T07:16:16,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:16,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815036785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:16,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:16,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:16,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:16,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:16,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815036788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:16,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-10T07:16:16,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:16,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:16,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:16,811 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/005f222920cf4d5da31284acd9af52a7 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/005f222920cf4d5da31284acd9af52a7 2024-12-10T07:16:16,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742210_1386 (size=12001) 2024-12-10T07:16:16,828 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6351062f9f9ac7f3c520d90dabe1dd24/C of 6351062f9f9ac7f3c520d90dabe1dd24 into 005f222920cf4d5da31284acd9af52a7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:16,828 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:16,828 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., storeName=6351062f9f9ac7f3c520d90dabe1dd24/C, priority=13, startTime=1733814976223; duration=0sec 2024-12-10T07:16:16,829 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:16,829 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6351062f9f9ac7f3c520d90dabe1dd24:C 2024-12-10T07:16:16,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:16,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48868 deadline: 1733815036835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:16,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:16,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48848 deadline: 1733815036835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:16,842 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 6351062f9f9ac7f3c520d90dabe1dd24 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:16,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:16,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:16,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:16,843 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 6351062f9f9ac7f3c520d90dabe1dd24 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:16,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:16,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:16,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:16,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:16,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48852 deadline: 1733815036844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:16,855 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 6351062f9f9ac7f3c520d90dabe1dd24 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:16,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:16,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:16,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:17,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815037094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:17,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815037105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:17,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:17,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:17,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:17,221 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/100bc14414684003ab8acf377c2039e6 2024-12-10T07:16:17,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/706ef29a0a12425daca0b004ddeadb6e is 50, key is test_row_0/C:col10/1733814975786/Put/seqid=0 2024-12-10T07:16:17,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742211_1387 (size=12001) 2024-12-10T07:16:17,241 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/706ef29a0a12425daca0b004ddeadb6e 2024-12-10T07:16:17,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/63b861846f1a438f9ce1eba8cba99ab2 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/63b861846f1a438f9ce1eba8cba99ab2 2024-12-10T07:16:17,249 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/63b861846f1a438f9ce1eba8cba99ab2, entries=150, sequenceid=68, filesize=30.2 K 2024-12-10T07:16:17,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/100bc14414684003ab8acf377c2039e6 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/100bc14414684003ab8acf377c2039e6 2024-12-10T07:16:17,255 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/100bc14414684003ab8acf377c2039e6, entries=150, sequenceid=68, filesize=11.7 K 2024-12-10T07:16:17,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/706ef29a0a12425daca0b004ddeadb6e as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/706ef29a0a12425daca0b004ddeadb6e 2024-12-10T07:16:17,261 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/706ef29a0a12425daca0b004ddeadb6e, entries=150, sequenceid=68, filesize=11.7 K 2024-12-10T07:16:17,262 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 6351062f9f9ac7f3c520d90dabe1dd24 in 1010ms, sequenceid=68, compaction requested=false 2024-12-10T07:16:17,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:17,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:17,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-10T07:16:17,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-10T07:16:17,265 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-10T07:16:17,265 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6250 sec 2024-12-10T07:16:17,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.6320 sec 2024-12-10T07:16:17,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:17,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6351062f9f9ac7f3c520d90dabe1dd24 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T07:16:17,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=A 2024-12-10T07:16:17,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:17,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=B 2024-12-10T07:16:17,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:17,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=C 2024-12-10T07:16:17,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:17,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a57585cfcf074c3ea174484207950769_6351062f9f9ac7f3c520d90dabe1dd24 is 50, key is test_row_0/A:col10/1733814977607/Put/seqid=0 2024-12-10T07:16:17,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742212_1388 (size=14594) 2024-12-10T07:16:17,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:17,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815037664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:17,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:17,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815037669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:17,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:17,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:17,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:17,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:17,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815037774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:17,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:17,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815037785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:17,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:17,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:17,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-10T07:16:17,805 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:16:17,805 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:16:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-10T07:16:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-10T07:16:17,807 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:16:17,807 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:16:17,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:16:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-10T07:16:17,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-10T07:16:17,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:17,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:17,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:17,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:17,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:18,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815037994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:18,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:18,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:18,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:18,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815037994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:18,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:18,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:18,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:18,035 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:18,037 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a57585cfcf074c3ea174484207950769_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a57585cfcf074c3ea174484207950769_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:18,038 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/26eff80184bc4015957edd1deeb3315c, store: [table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:18,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/26eff80184bc4015957edd1deeb3315c is 175, key is test_row_0/A:col10/1733814977607/Put/seqid=0 2024-12-10T07:16:18,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742213_1389 (size=39549) 2024-12-10T07:16:18,051 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/26eff80184bc4015957edd1deeb3315c 2024-12-10T07:16:18,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/de31edcef07841caaf3656d984960a27 is 50, key is test_row_0/B:col10/1733814977607/Put/seqid=0 2024-12-10T07:16:18,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742214_1390 (size=12001) 2024-12-10T07:16:18,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-10T07:16:18,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:18,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:18,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:18,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:18,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-10T07:16:18,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-10T07:16:18,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:18,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:18,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:18,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:18,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:18,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815038305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:18,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815038305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:18,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:18,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:18,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:18,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:18,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:18,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:18,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-10T07:16:18,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:18,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:18,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:18,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:18,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:18,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:18,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-10T07:16:18,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/de31edcef07841caaf3656d984960a27 2024-12-10T07:16:18,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/36e0d50a233d49609acd1653f48a9b73 is 50, key is test_row_0/C:col10/1733814977607/Put/seqid=0 2024-12-10T07:16:18,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742215_1391 (size=12001) 2024-12-10T07:16:18,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-10T07:16:18,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:18,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:18,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:18,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:18,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:18,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-10T07:16:18,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:18,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:18,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:18,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:18,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:18,826 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T07:16:18,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815038824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:18,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:18,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:18,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815038825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:18,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:18,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:18,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:18,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:18,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48848 deadline: 1733815038855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:18,858 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 6351062f9f9ac7f3c520d90dabe1dd24 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:18,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:18,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:18,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:18,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/36e0d50a233d49609acd1653f48a9b73 2024-12-10T07:16:18,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48868 deadline: 1733815038865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:18,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48852 deadline: 1733815038865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:18,873 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 6351062f9f9ac7f3c520d90dabe1dd24 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:18,873 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 6351062f9f9ac7f3c520d90dabe1dd24 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:18,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/26eff80184bc4015957edd1deeb3315c as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/26eff80184bc4015957edd1deeb3315c 2024-12-10T07:16:18,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/26eff80184bc4015957edd1deeb3315c, entries=200, sequenceid=86, filesize=38.6 K 2024-12-10T07:16:18,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/de31edcef07841caaf3656d984960a27 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/de31edcef07841caaf3656d984960a27 2024-12-10T07:16:18,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-10T07:16:18,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:18,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:18,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:18,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:18,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:18,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/de31edcef07841caaf3656d984960a27, entries=150, sequenceid=86, filesize=11.7 K 2024-12-10T07:16:18,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/36e0d50a233d49609acd1653f48a9b73 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/36e0d50a233d49609acd1653f48a9b73 2024-12-10T07:16:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:18,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/36e0d50a233d49609acd1653f48a9b73, entries=150, sequenceid=86, filesize=11.7 K 2024-12-10T07:16:18,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 6351062f9f9ac7f3c520d90dabe1dd24 in 1283ms, sequenceid=86, compaction requested=true 2024-12-10T07:16:18,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:18,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6351062f9f9ac7f3c520d90dabe1dd24:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:16:18,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:18,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6351062f9f9ac7f3c520d90dabe1dd24:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:16:18,891 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:16:18,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:18,891 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:16:18,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6351062f9f9ac7f3c520d90dabe1dd24:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:16:18,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:16:18,892 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:16:18,892 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:16:18,892 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 6351062f9f9ac7f3c520d90dabe1dd24/B is initiating minor compaction (all files) 2024-12-10T07:16:18,892 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 6351062f9f9ac7f3c520d90dabe1dd24/A is initiating minor compaction (all files) 2024-12-10T07:16:18,892 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6351062f9f9ac7f3c520d90dabe1dd24/B in TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:18,892 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6351062f9f9ac7f3c520d90dabe1dd24/A in TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:18,892 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/1afbe93976f947da8cbe8fdcb5aca701, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/100bc14414684003ab8acf377c2039e6, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/de31edcef07841caaf3656d984960a27] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp, totalSize=35.3 K 2024-12-10T07:16:18,892 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/58309db390694cd2901f6df07a2a5646, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/63b861846f1a438f9ce1eba8cba99ab2, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/26eff80184bc4015957edd1deeb3315c] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp, totalSize=99.2 K 2024-12-10T07:16:18,892 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:18,892 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. files: [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/58309db390694cd2901f6df07a2a5646, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/63b861846f1a438f9ce1eba8cba99ab2, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/26eff80184bc4015957edd1deeb3315c] 2024-12-10T07:16:18,892 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 1afbe93976f947da8cbe8fdcb5aca701, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733814974670 2024-12-10T07:16:18,892 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58309db390694cd2901f6df07a2a5646, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733814974670 2024-12-10T07:16:18,893 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 100bc14414684003ab8acf377c2039e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733814975744 2024-12-10T07:16:18,893 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63b861846f1a438f9ce1eba8cba99ab2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733814975744 2024-12-10T07:16:18,893 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting de31edcef07841caaf3656d984960a27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733814976446 2024-12-10T07:16:18,893 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26eff80184bc4015957edd1deeb3315c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733814976446 2024-12-10T07:16:18,902 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:18,903 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6351062f9f9ac7f3c520d90dabe1dd24#B#compaction#332 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:18,903 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/952d41684fe642b5bc4860effde55e89 is 50, key is test_row_0/B:col10/1733814977607/Put/seqid=0 2024-12-10T07:16:18,906 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210d42b0f0cfd0b47da998e33a9f860ad7d_6351062f9f9ac7f3c520d90dabe1dd24 store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:18,908 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210d42b0f0cfd0b47da998e33a9f860ad7d_6351062f9f9ac7f3c520d90dabe1dd24, store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:18,908 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d42b0f0cfd0b47da998e33a9f860ad7d_6351062f9f9ac7f3c520d90dabe1dd24 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:18,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742216_1392 (size=12207) 2024-12-10T07:16:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-10T07:16:18,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742217_1393 (size=4469) 2024-12-10T07:16:19,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-10T07:16:19,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:19,035 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 6351062f9f9ac7f3c520d90dabe1dd24 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T07:16:19,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=A 2024-12-10T07:16:19,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:19,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=B 2024-12-10T07:16:19,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:19,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=C 2024-12-10T07:16:19,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:19,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121084ae884364e5451e9b88791dd3f132c8_6351062f9f9ac7f3c520d90dabe1dd24 is 50, key is test_row_0/A:col10/1733814977652/Put/seqid=0 2024-12-10T07:16:19,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742218_1394 (size=12154) 2024-12-10T07:16:19,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,076 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121084ae884364e5451e9b88791dd3f132c8_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121084ae884364e5451e9b88791dd3f132c8_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:19,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/7dea415dce57440c94f1df77887b8f64, store: [table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:19,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/7dea415dce57440c94f1df77887b8f64 is 175, key is test_row_0/A:col10/1733814977652/Put/seqid=0 2024-12-10T07:16:19,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742219_1395 (size=30955) 2024-12-10T07:16:19,348 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/952d41684fe642b5bc4860effde55e89 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/952d41684fe642b5bc4860effde55e89 2024-12-10T07:16:19,353 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6351062f9f9ac7f3c520d90dabe1dd24#A#compaction#333 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:19,354 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/a068f16540a840c29c2a43ae0bf2c812 is 175, key is test_row_0/A:col10/1733814977607/Put/seqid=0 2024-12-10T07:16:19,371 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6351062f9f9ac7f3c520d90dabe1dd24/B of 6351062f9f9ac7f3c520d90dabe1dd24 into 952d41684fe642b5bc4860effde55e89(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:19,371 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:19,371 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., storeName=6351062f9f9ac7f3c520d90dabe1dd24/B, priority=13, startTime=1733814978891; duration=0sec 2024-12-10T07:16:19,371 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:16:19,371 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6351062f9f9ac7f3c520d90dabe1dd24:B 2024-12-10T07:16:19,371 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:16:19,373 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:16:19,373 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 6351062f9f9ac7f3c520d90dabe1dd24/C is initiating minor compaction (all files) 2024-12-10T07:16:19,373 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6351062f9f9ac7f3c520d90dabe1dd24/C in TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:19,373 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/005f222920cf4d5da31284acd9af52a7, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/706ef29a0a12425daca0b004ddeadb6e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/36e0d50a233d49609acd1653f48a9b73] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp, totalSize=35.3 K 2024-12-10T07:16:19,374 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 005f222920cf4d5da31284acd9af52a7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733814974670 2024-12-10T07:16:19,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742220_1396 (size=31161) 2024-12-10T07:16:19,375 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 706ef29a0a12425daca0b004ddeadb6e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733814975744 2024-12-10T07:16:19,375 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 36e0d50a233d49609acd1653f48a9b73, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733814976446 2024-12-10T07:16:19,378 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/a068f16540a840c29c2a43ae0bf2c812 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/a068f16540a840c29c2a43ae0bf2c812 2024-12-10T07:16:19,389 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6351062f9f9ac7f3c520d90dabe1dd24#C#compaction#335 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:19,391 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/042c2bc8f6c94ce688a0f1a931a1a0d7 is 50, key is test_row_0/C:col10/1733814977607/Put/seqid=0 2024-12-10T07:16:19,405 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6351062f9f9ac7f3c520d90dabe1dd24/A of 6351062f9f9ac7f3c520d90dabe1dd24 into a068f16540a840c29c2a43ae0bf2c812(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:19,405 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:19,405 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., storeName=6351062f9f9ac7f3c520d90dabe1dd24/A, priority=13, startTime=1733814978891; duration=0sec 2024-12-10T07:16:19,405 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:19,405 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6351062f9f9ac7f3c520d90dabe1dd24:A 2024-12-10T07:16:19,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742221_1397 (size=12207) 2024-12-10T07:16:19,506 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=103, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/7dea415dce57440c94f1df77887b8f64 2024-12-10T07:16:19,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/f0717babdb8448e2a552bbd866694a29 is 50, key is test_row_0/B:col10/1733814977652/Put/seqid=0 2024-12-10T07:16:19,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742222_1398 (size=12001) 2024-12-10T07:16:19,559 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/f0717babdb8448e2a552bbd866694a29 2024-12-10T07:16:19,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/c355d924c3d449d29d4c9367786ca88b is 50, key is test_row_0/C:col10/1733814977652/Put/seqid=0 2024-12-10T07:16:19,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742223_1399 (size=12001) 2024-12-10T07:16:19,627 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/c355d924c3d449d29d4c9367786ca88b 2024-12-10T07:16:19,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/7dea415dce57440c94f1df77887b8f64 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/7dea415dce57440c94f1df77887b8f64 2024-12-10T07:16:19,647 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/7dea415dce57440c94f1df77887b8f64, entries=150, sequenceid=103, filesize=30.2 K 2024-12-10T07:16:19,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/f0717babdb8448e2a552bbd866694a29 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/f0717babdb8448e2a552bbd866694a29 2024-12-10T07:16:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,653 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/f0717babdb8448e2a552bbd866694a29, entries=150, sequenceid=103, filesize=11.7 K 2024-12-10T07:16:19,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/c355d924c3d449d29d4c9367786ca88b as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/c355d924c3d449d29d4c9367786ca88b 2024-12-10T07:16:19,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,659 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/c355d924c3d449d29d4c9367786ca88b, entries=150, sequenceid=103, filesize=11.7 K 2024-12-10T07:16:19,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,665 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 6351062f9f9ac7f3c520d90dabe1dd24 in 629ms, sequenceid=103, compaction requested=false 2024-12-10T07:16:19,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:19,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:19,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-10T07:16:19,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-10T07:16:19,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,674 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-10T07:16:19,674 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8620 sec 2024-12-10T07:16:19,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.8690 sec 2024-12-10T07:16:19,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,829 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/042c2bc8f6c94ce688a0f1a931a1a0d7 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/042c2bc8f6c94ce688a0f1a931a1a0d7 2024-12-10T07:16:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,838 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6351062f9f9ac7f3c520d90dabe1dd24/C of 6351062f9f9ac7f3c520d90dabe1dd24 into 042c2bc8f6c94ce688a0f1a931a1a0d7(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:19,838 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:19,838 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., storeName=6351062f9f9ac7f3c520d90dabe1dd24/C, priority=13, startTime=1733814978891; duration=0sec 2024-12-10T07:16:19,838 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:19,838 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6351062f9f9ac7f3c520d90dabe1dd24:C 2024-12-10T07:16:19,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-10T07:16:19,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,955 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:16:19,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,956 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:16:19,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-10T07:16:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T07:16:19,958 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:16:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,959 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:16:19,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:16:19,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:19,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T07:16:20,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-10T07:16:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:20,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,118 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 6351062f9f9ac7f3c520d90dabe1dd24 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-10T07:16:20,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=A 2024-12-10T07:16:20,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:20,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=B 2024-12-10T07:16:20,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:20,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=C 2024-12-10T07:16:20,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:20,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210dad699dbf2804e07b74b66661a8fe6d9_6351062f9f9ac7f3c520d90dabe1dd24 is 50, key is test_row_1/A:col10/1733814980059/Put/seqid=0 2024-12-10T07:16:20,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742224_1400 (size=12154) 2024-12-10T07:16:20,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,182 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210dad699dbf2804e07b74b66661a8fe6d9_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210dad699dbf2804e07b74b66661a8fe6d9_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:20,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/92f90ef305ae42a0ae9bab37e6735ce2, store: [table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:20,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/92f90ef305ae42a0ae9bab37e6735ce2 is 175, key is test_row_1/A:col10/1733814980059/Put/seqid=0 2024-12-10T07:16:20,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742225_1401 (size=30951) 2024-12-10T07:16:20,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,200 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=113, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/92f90ef305ae42a0ae9bab37e6735ce2 2024-12-10T07:16:20,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:20,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:20,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/ba95a51b17c248ecabb971f2a1b7be46 is 50, key is test_row_1/B:col10/1733814980059/Put/seqid=0 2024-12-10T07:16:20,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742226_1402 (size=9657) 2024-12-10T07:16:20,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,272 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/ba95a51b17c248ecabb971f2a1b7be46 2024-12-10T07:16:20,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T07:16:20,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/68c2f5882e394796a087e3d30a662b54 is 50, key is test_row_1/C:col10/1733814980059/Put/seqid=0 2024-12-10T07:16:20,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742227_1403 (size=9657) 2024-12-10T07:16:20,323 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/68c2f5882e394796a087e3d30a662b54 2024-12-10T07:16:20,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/92f90ef305ae42a0ae9bab37e6735ce2 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/92f90ef305ae42a0ae9bab37e6735ce2 2024-12-10T07:16:20,348 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/92f90ef305ae42a0ae9bab37e6735ce2, entries=150, sequenceid=113, filesize=30.2 K 2024-12-10T07:16:20,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/ba95a51b17c248ecabb971f2a1b7be46 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/ba95a51b17c248ecabb971f2a1b7be46 2024-12-10T07:16:20,361 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/ba95a51b17c248ecabb971f2a1b7be46, entries=100, sequenceid=113, filesize=9.4 K 2024-12-10T07:16:20,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/68c2f5882e394796a087e3d30a662b54 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/68c2f5882e394796a087e3d30a662b54 2024-12-10T07:16:20,372 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/68c2f5882e394796a087e3d30a662b54, entries=100, sequenceid=113, filesize=9.4 K 2024-12-10T07:16:20,373 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=100.63 KB/103050 for 6351062f9f9ac7f3c520d90dabe1dd24 in 255ms, sequenceid=113, compaction requested=true 2024-12-10T07:16:20,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:20,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:20,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-10T07:16:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:20,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6351062f9f9ac7f3c520d90dabe1dd24 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-10T07:16:20,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=A 2024-12-10T07:16:20,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:20,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=B 2024-12-10T07:16:20,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:20,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=C 2024-12-10T07:16:20,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-10T07:16:20,378 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-10T07:16:20,378 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 416 msec 2024-12-10T07:16:20,380 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 423 msec 2024-12-10T07:16:20,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f753a0fb9c7249558de262b6e20b8ae2_6351062f9f9ac7f3c520d90dabe1dd24 is 50, key is test_row_0/A:col10/1733814980371/Put/seqid=0 2024-12-10T07:16:20,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742228_1404 (size=12204) 2024-12-10T07:16:20,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815040455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:20,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:20,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:20,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:20,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815040459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:20,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:20,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:20,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:20,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815040576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:20,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:20,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815040576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-10T07:16:20,585 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:16:20,586 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:16:20,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-10T07:16:20,587 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:16:20,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T07:16:20,588 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:16:20,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:16:20,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T07:16:20,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T07:16:20,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:20,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:20,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:20,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:20,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:20,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:20,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:20,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815040795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:20,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:20,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:20,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:20,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:20,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815040795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:20,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:20,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:20,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:20,810 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:20,814 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210f753a0fb9c7249558de262b6e20b8ae2_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f753a0fb9c7249558de262b6e20b8ae2_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:20,815 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/c9debd96edd2449c9306576198b6773d, store: [table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:20,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/c9debd96edd2449c9306576198b6773d is 175, key is test_row_0/A:col10/1733814980371/Put/seqid=0 2024-12-10T07:16:20,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742229_1405 (size=31005) 2024-12-10T07:16:20,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T07:16:20,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:20,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:20,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:20,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:20,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:20,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T07:16:21,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T07:16:21,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:21,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:21,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:21,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:21,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:21,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:21,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:21,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815041114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:21,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:21,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815041115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:21,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T07:16:21,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:21,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:21,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:21,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:21,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:21,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T07:16:21,221 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/c9debd96edd2449c9306576198b6773d 2024-12-10T07:16:21,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/b171910e6ccf43fdb5e05a9605e9880f is 50, key is test_row_0/B:col10/1733814980371/Put/seqid=0 2024-12-10T07:16:21,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742230_1406 (size=12051) 2024-12-10T07:16:21,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T07:16:21,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:21,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:21,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:21,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:21,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:21,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:21,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T07:16:21,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:21,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:21,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:21,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:21,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:21,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:21,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:21,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:21,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815041625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:21,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815041625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:21,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/b171910e6ccf43fdb5e05a9605e9880f 2024-12-10T07:16:21,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T07:16:21,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:21,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:21,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:21,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:21,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:21,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:21,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/e660ba6182f1486fbee19f9f79e683b4 is 50, key is test_row_0/C:col10/1733814980371/Put/seqid=0 2024-12-10T07:16:21,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T07:16:21,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742231_1407 (size=12051) 2024-12-10T07:16:21,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T07:16:21,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:21,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:21,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:21,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:21,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:21,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T07:16:21,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:21,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:21,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:21,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:21,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:22,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/e660ba6182f1486fbee19f9f79e683b4 2024-12-10T07:16:22,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T07:16:22,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:22,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:22,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:22,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:22,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:22,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/c9debd96edd2449c9306576198b6773d as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/c9debd96edd2449c9306576198b6773d 2024-12-10T07:16:22,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/c9debd96edd2449c9306576198b6773d, entries=150, sequenceid=132, filesize=30.3 K 2024-12-10T07:16:22,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/b171910e6ccf43fdb5e05a9605e9880f as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/b171910e6ccf43fdb5e05a9605e9880f 2024-12-10T07:16:22,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/b171910e6ccf43fdb5e05a9605e9880f, entries=150, sequenceid=132, filesize=11.8 K 2024-12-10T07:16:22,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/e660ba6182f1486fbee19f9f79e683b4 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/e660ba6182f1486fbee19f9f79e683b4 2024-12-10T07:16:22,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/e660ba6182f1486fbee19f9f79e683b4, entries=150, sequenceid=132, filesize=11.8 K 2024-12-10T07:16:22,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6351062f9f9ac7f3c520d90dabe1dd24 in 1877ms, sequenceid=132, compaction requested=true 2024-12-10T07:16:22,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:22,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6351062f9f9ac7f3c520d90dabe1dd24:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:16:22,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:22,250 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:16:22,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6351062f9f9ac7f3c520d90dabe1dd24:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:16:22,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:16:22,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6351062f9f9ac7f3c520d90dabe1dd24:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:16:22,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:16:22,250 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:16:22,272 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124072 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:16:22,272 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 6351062f9f9ac7f3c520d90dabe1dd24/A is initiating minor compaction (all files) 2024-12-10T07:16:22,272 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6351062f9f9ac7f3c520d90dabe1dd24/A in TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:22,272 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/a068f16540a840c29c2a43ae0bf2c812, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/7dea415dce57440c94f1df77887b8f64, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/92f90ef305ae42a0ae9bab37e6735ce2, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/c9debd96edd2449c9306576198b6773d] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp, totalSize=121.2 K 2024-12-10T07:16:22,272 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:22,272 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. files: [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/a068f16540a840c29c2a43ae0bf2c812, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/7dea415dce57440c94f1df77887b8f64, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/92f90ef305ae42a0ae9bab37e6735ce2, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/c9debd96edd2449c9306576198b6773d] 2024-12-10T07:16:22,273 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45916 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:16:22,273 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 6351062f9f9ac7f3c520d90dabe1dd24/B is initiating minor compaction (all files) 2024-12-10T07:16:22,273 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting a068f16540a840c29c2a43ae0bf2c812, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733814976446 2024-12-10T07:16:22,273 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6351062f9f9ac7f3c520d90dabe1dd24/B in TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:22,273 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/952d41684fe642b5bc4860effde55e89, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/f0717babdb8448e2a552bbd866694a29, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/ba95a51b17c248ecabb971f2a1b7be46, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/b171910e6ccf43fdb5e05a9605e9880f] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp, totalSize=44.8 K 2024-12-10T07:16:22,273 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7dea415dce57440c94f1df77887b8f64, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733814977652 2024-12-10T07:16:22,273 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 952d41684fe642b5bc4860effde55e89, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733814976446 2024-12-10T07:16:22,274 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92f90ef305ae42a0ae9bab37e6735ce2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733814979951 2024-12-10T07:16:22,274 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting f0717babdb8448e2a552bbd866694a29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733814977652 2024-12-10T07:16:22,274 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9debd96edd2449c9306576198b6773d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733814980322 2024-12-10T07:16:22,274 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting ba95a51b17c248ecabb971f2a1b7be46, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733814980059 2024-12-10T07:16:22,275 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting b171910e6ccf43fdb5e05a9605e9880f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733814980322 2024-12-10T07:16:22,297 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:22,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-10T07:16:22,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:22,307 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 6351062f9f9ac7f3c520d90dabe1dd24 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-10T07:16:22,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=A 2024-12-10T07:16:22,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:22,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=B 2024-12-10T07:16:22,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:22,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=C 2024-12-10T07:16:22,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:22,315 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6351062f9f9ac7f3c520d90dabe1dd24#B#compaction#345 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:22,315 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/835024302ea74862bc2be89944d77303 is 50, key is test_row_0/B:col10/1733814980371/Put/seqid=0 2024-12-10T07:16:22,329 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121004c9206935f34b4e97ae1dac4c526791_6351062f9f9ac7f3c520d90dabe1dd24 store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:22,331 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121004c9206935f34b4e97ae1dac4c526791_6351062f9f9ac7f3c520d90dabe1dd24, store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:22,332 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121004c9206935f34b4e97ae1dac4c526791_6351062f9f9ac7f3c520d90dabe1dd24 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:22,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210251dc20942d441c4bcfb96279c7ba640_6351062f9f9ac7f3c520d90dabe1dd24 is 50, key is test_row_0/A:col10/1733814980447/Put/seqid=0 2024-12-10T07:16:22,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742232_1408 (size=12393) 2024-12-10T07:16:22,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742233_1409 (size=4469) 2024-12-10T07:16:22,430 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6351062f9f9ac7f3c520d90dabe1dd24#A#compaction#344 average throughput is 0.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:22,431 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/af4361d0cd1847a2aafabd242b2fce0e is 175, key is test_row_0/A:col10/1733814980371/Put/seqid=0 2024-12-10T07:16:22,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742234_1410 (size=12304) 2024-12-10T07:16:22,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:22,468 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210251dc20942d441c4bcfb96279c7ba640_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210251dc20942d441c4bcfb96279c7ba640_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:22,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/853728224f414732af03b746fe1ff4ba, store: [table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:22,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/853728224f414732af03b746fe1ff4ba is 175, key is test_row_0/A:col10/1733814980447/Put/seqid=0 2024-12-10T07:16:22,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742235_1411 (size=31347) 2024-12-10T07:16:22,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742236_1412 (size=31105) 2024-12-10T07:16:22,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T07:16:22,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815042746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:22,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:22,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815042748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:22,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:22,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:22,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:22,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:22,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:22,792 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/835024302ea74862bc2be89944d77303 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/835024302ea74862bc2be89944d77303 2024-12-10T07:16:22,802 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6351062f9f9ac7f3c520d90dabe1dd24/B of 6351062f9f9ac7f3c520d90dabe1dd24 into 835024302ea74862bc2be89944d77303(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:22,803 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:22,803 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., storeName=6351062f9f9ac7f3c520d90dabe1dd24/B, priority=12, startTime=1733814982250; duration=0sec 2024-12-10T07:16:22,803 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:16:22,803 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6351062f9f9ac7f3c520d90dabe1dd24:B 2024-12-10T07:16:22,803 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:16:22,806 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45916 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:16:22,806 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 6351062f9f9ac7f3c520d90dabe1dd24/C is initiating minor compaction (all files) 2024-12-10T07:16:22,806 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6351062f9f9ac7f3c520d90dabe1dd24/C in TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:22,806 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/042c2bc8f6c94ce688a0f1a931a1a0d7, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/c355d924c3d449d29d4c9367786ca88b, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/68c2f5882e394796a087e3d30a662b54, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/e660ba6182f1486fbee19f9f79e683b4] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp, totalSize=44.8 K 2024-12-10T07:16:22,806 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 042c2bc8f6c94ce688a0f1a931a1a0d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733814976446 2024-12-10T07:16:22,807 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting c355d924c3d449d29d4c9367786ca88b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733814977652 2024-12-10T07:16:22,808 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 68c2f5882e394796a087e3d30a662b54, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733814980059 2024-12-10T07:16:22,811 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting e660ba6182f1486fbee19f9f79e683b4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733814980322 2024-12-10T07:16:22,818 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6351062f9f9ac7f3c520d90dabe1dd24#C#compaction#347 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:22,819 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/c69253f3e0ba46418457da6cc30e0a0b is 50, key is test_row_0/C:col10/1733814980371/Put/seqid=0 2024-12-10T07:16:22,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742237_1413 (size=12393) 2024-12-10T07:16:22,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815042856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:22,862 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/c69253f3e0ba46418457da6cc30e0a0b as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/c69253f3e0ba46418457da6cc30e0a0b 2024-12-10T07:16:22,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815042858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:22,868 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6351062f9f9ac7f3c520d90dabe1dd24/C of 6351062f9f9ac7f3c520d90dabe1dd24 into c69253f3e0ba46418457da6cc30e0a0b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:22,868 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:22,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:22,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:22,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:22,869 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., storeName=6351062f9f9ac7f3c520d90dabe1dd24/C, priority=12, startTime=1733814982250; duration=0sec 2024-12-10T07:16:22,869 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:22,869 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6351062f9f9ac7f3c520d90dabe1dd24:C 2024-12-10T07:16:22,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48852 deadline: 1733815042886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:22,896 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 6351062f9f9ac7f3c520d90dabe1dd24 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:22,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:22,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:22,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:22,897 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/af4361d0cd1847a2aafabd242b2fce0e as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/af4361d0cd1847a2aafabd242b2fce0e 2024-12-10T07:16:22,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48848 deadline: 1733815042897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:22,906 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 6351062f9f9ac7f3c520d90dabe1dd24 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:22,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:22,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:22,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:22,910 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6351062f9f9ac7f3c520d90dabe1dd24/A of 6351062f9f9ac7f3c520d90dabe1dd24 into af4361d0cd1847a2aafabd242b2fce0e(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:22,910 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:22,910 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., storeName=6351062f9f9ac7f3c520d90dabe1dd24/A, priority=12, startTime=1733814982250; duration=0sec 2024-12-10T07:16:22,910 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:22,910 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6351062f9f9ac7f3c520d90dabe1dd24:A 2024-12-10T07:16:22,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48868 deadline: 1733815042906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:22,912 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 6351062f9f9ac7f3c520d90dabe1dd24 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:22,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:22,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:22,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:22,925 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/853728224f414732af03b746fe1ff4ba 2024-12-10T07:16:22,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/6743b16aa8bd48e2afd7b1543c3ce981 is 50, key is test_row_0/B:col10/1733814980447/Put/seqid=0 2024-12-10T07:16:22,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742238_1414 (size=12151) 2024-12-10T07:16:22,961 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/6743b16aa8bd48e2afd7b1543c3ce981 2024-12-10T07:16:22,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/bfb25d07914643ec9438cfc379183521 is 50, key is test_row_0/C:col10/1733814980447/Put/seqid=0 2024-12-10T07:16:23,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742239_1415 (size=12151) 2024-12-10T07:16:23,006 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/bfb25d07914643ec9438cfc379183521 2024-12-10T07:16:23,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/853728224f414732af03b746fe1ff4ba as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/853728224f414732af03b746fe1ff4ba 2024-12-10T07:16:23,019 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/853728224f414732af03b746fe1ff4ba, entries=150, sequenceid=145, filesize=30.4 K 2024-12-10T07:16:23,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/6743b16aa8bd48e2afd7b1543c3ce981 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/6743b16aa8bd48e2afd7b1543c3ce981 2024-12-10T07:16:23,037 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/6743b16aa8bd48e2afd7b1543c3ce981, entries=150, sequenceid=145, filesize=11.9 K 2024-12-10T07:16:23,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/bfb25d07914643ec9438cfc379183521 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/bfb25d07914643ec9438cfc379183521 2024-12-10T07:16:23,045 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/bfb25d07914643ec9438cfc379183521, entries=150, sequenceid=145, filesize=11.9 K 2024-12-10T07:16:23,046 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6351062f9f9ac7f3c520d90dabe1dd24 in 739ms, sequenceid=145, compaction requested=false 2024-12-10T07:16:23,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:23,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:23,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-10T07:16:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-10T07:16:23,050 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-10T07:16:23,050 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4590 sec 2024-12-10T07:16:23,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.4650 sec 2024-12-10T07:16:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:23,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6351062f9f9ac7f3c520d90dabe1dd24 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-10T07:16:23,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=A 2024-12-10T07:16:23,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:23,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=B 2024-12-10T07:16:23,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:23,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=C 2024-12-10T07:16:23,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:23,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412102db89eeb1a3f45ccb3d24348730afbd2_6351062f9f9ac7f3c520d90dabe1dd24 is 50, key is test_row_0/A:col10/1733814982745/Put/seqid=0 2024-12-10T07:16:23,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742240_1416 (size=14794) 2024-12-10T07:16:23,095 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:23,107 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412102db89eeb1a3f45ccb3d24348730afbd2_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412102db89eeb1a3f45ccb3d24348730afbd2_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:23,108 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/97c0be489cd649e2b26e18c7d8605f67, store: [table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:23,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/97c0be489cd649e2b26e18c7d8605f67 is 175, key is test_row_0/A:col10/1733814982745/Put/seqid=0 2024-12-10T07:16:23,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742241_1417 (size=39749) 2024-12-10T07:16:23,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815043134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:23,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815043134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:23,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815043245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:23,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815043245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:23,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:23,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815043466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:23,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:23,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:23,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:23,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:23,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815043466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:23,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:23,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:23,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:23,535 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/97c0be489cd649e2b26e18c7d8605f67 2024-12-10T07:16:23,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/0cada39f52e6477e99c3a773651f212d is 50, key is test_row_0/B:col10/1733814982745/Put/seqid=0 2024-12-10T07:16:23,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742242_1418 (size=12151) 2024-12-10T07:16:23,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815043785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:23,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:23,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:23,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:23,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815043785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:23,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:23,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:23,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:23,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/0cada39f52e6477e99c3a773651f212d 2024-12-10T07:16:23,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/2e3710bb5ae6469f8169617a4dcaa419 is 50, key is test_row_0/C:col10/1733814982745/Put/seqid=0 2024-12-10T07:16:24,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742243_1419 (size=12151) 2024-12-10T07:16:24,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:24,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815044295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:24,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:24,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:24,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:24,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815044295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:24,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:24,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:24,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:24,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:24,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/2e3710bb5ae6469f8169617a4dcaa419 2024-12-10T07:16:24,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/97c0be489cd649e2b26e18c7d8605f67 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/97c0be489cd649e2b26e18c7d8605f67 2024-12-10T07:16:24,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/97c0be489cd649e2b26e18c7d8605f67, entries=200, sequenceid=168, filesize=38.8 K 2024-12-10T07:16:24,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/0cada39f52e6477e99c3a773651f212d as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/0cada39f52e6477e99c3a773651f212d 2024-12-10T07:16:24,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/0cada39f52e6477e99c3a773651f212d, entries=150, sequenceid=168, filesize=11.9 K 2024-12-10T07:16:24,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/2e3710bb5ae6469f8169617a4dcaa419 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/2e3710bb5ae6469f8169617a4dcaa419 2024-12-10T07:16:24,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/2e3710bb5ae6469f8169617a4dcaa419, entries=150, sequenceid=168, filesize=11.9 K 2024-12-10T07:16:24,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6351062f9f9ac7f3c520d90dabe1dd24 in 1375ms, sequenceid=168, compaction requested=true 2024-12-10T07:16:24,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:24,445 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:16:24,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6351062f9f9ac7f3c520d90dabe1dd24:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:16:24,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:24,445 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:16:24,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6351062f9f9ac7f3c520d90dabe1dd24:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:16:24,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:24,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6351062f9f9ac7f3c520d90dabe1dd24:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:16:24,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:16:24,451 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102201 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:16:24,451 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 6351062f9f9ac7f3c520d90dabe1dd24/A is initiating minor compaction (all files) 2024-12-10T07:16:24,452 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6351062f9f9ac7f3c520d90dabe1dd24/A in TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:24,452 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/af4361d0cd1847a2aafabd242b2fce0e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/853728224f414732af03b746fe1ff4ba, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/97c0be489cd649e2b26e18c7d8605f67] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp, totalSize=99.8 K 2024-12-10T07:16:24,452 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:24,452 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. files: [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/af4361d0cd1847a2aafabd242b2fce0e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/853728224f414732af03b746fe1ff4ba, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/97c0be489cd649e2b26e18c7d8605f67] 2024-12-10T07:16:24,452 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36695 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:16:24,453 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 6351062f9f9ac7f3c520d90dabe1dd24/B is initiating minor compaction (all files) 2024-12-10T07:16:24,453 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6351062f9f9ac7f3c520d90dabe1dd24/B in TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:24,453 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/835024302ea74862bc2be89944d77303, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/6743b16aa8bd48e2afd7b1543c3ce981, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/0cada39f52e6477e99c3a773651f212d] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp, totalSize=35.8 K 2024-12-10T07:16:24,453 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting af4361d0cd1847a2aafabd242b2fce0e, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733814980322 2024-12-10T07:16:24,453 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 835024302ea74862bc2be89944d77303, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733814980322 2024-12-10T07:16:24,453 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 853728224f414732af03b746fe1ff4ba, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733814980441 2024-12-10T07:16:24,454 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97c0be489cd649e2b26e18c7d8605f67, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733814982711 2024-12-10T07:16:24,454 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 6743b16aa8bd48e2afd7b1543c3ce981, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733814980441 2024-12-10T07:16:24,454 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cada39f52e6477e99c3a773651f212d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733814982743 2024-12-10T07:16:24,470 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:24,471 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6351062f9f9ac7f3c520d90dabe1dd24#B#compaction#353 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:24,471 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/39b262a38a7e4e86b809a3aa8aeab52f is 50, key is test_row_0/B:col10/1733814982745/Put/seqid=0 2024-12-10T07:16:24,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742244_1420 (size=12595) 2024-12-10T07:16:24,488 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210bd68998526e44aba9491a2e422ecc2e5_6351062f9f9ac7f3c520d90dabe1dd24 store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:24,492 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/39b262a38a7e4e86b809a3aa8aeab52f as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/39b262a38a7e4e86b809a3aa8aeab52f 2024-12-10T07:16:24,493 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210bd68998526e44aba9491a2e422ecc2e5_6351062f9f9ac7f3c520d90dabe1dd24, store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:24,494 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210bd68998526e44aba9491a2e422ecc2e5_6351062f9f9ac7f3c520d90dabe1dd24 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:24,510 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6351062f9f9ac7f3c520d90dabe1dd24/B of 6351062f9f9ac7f3c520d90dabe1dd24 into 39b262a38a7e4e86b809a3aa8aeab52f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:24,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742245_1421 (size=4469) 2024-12-10T07:16:24,510 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:24,510 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., storeName=6351062f9f9ac7f3c520d90dabe1dd24/B, priority=13, startTime=1733814984445; duration=0sec 2024-12-10T07:16:24,510 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:16:24,510 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6351062f9f9ac7f3c520d90dabe1dd24:B 2024-12-10T07:16:24,510 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:16:24,512 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36695 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:16:24,512 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 6351062f9f9ac7f3c520d90dabe1dd24/C is initiating minor compaction (all files) 2024-12-10T07:16:24,512 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6351062f9f9ac7f3c520d90dabe1dd24/C in TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:24,512 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/c69253f3e0ba46418457da6cc30e0a0b, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/bfb25d07914643ec9438cfc379183521, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/2e3710bb5ae6469f8169617a4dcaa419] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp, totalSize=35.8 K 2024-12-10T07:16:24,514 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting c69253f3e0ba46418457da6cc30e0a0b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733814980322 2024-12-10T07:16:24,515 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6351062f9f9ac7f3c520d90dabe1dd24#A#compaction#354 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:24,515 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/ac0ff91e9a4146dba57bc0cb03931e4c is 175, key is test_row_0/A:col10/1733814982745/Put/seqid=0 2024-12-10T07:16:24,516 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting bfb25d07914643ec9438cfc379183521, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733814980441 2024-12-10T07:16:24,517 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e3710bb5ae6469f8169617a4dcaa419, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733814982743 2024-12-10T07:16:24,526 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6351062f9f9ac7f3c520d90dabe1dd24#C#compaction#355 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:24,526 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/d9c8bfcb059044e2935eae547fa5475a is 50, key is test_row_0/C:col10/1733814982745/Put/seqid=0 2024-12-10T07:16:24,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742246_1422 (size=31549) 2024-12-10T07:16:24,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742247_1423 (size=12595) 2024-12-10T07:16:24,541 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/ac0ff91e9a4146dba57bc0cb03931e4c as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/ac0ff91e9a4146dba57bc0cb03931e4c 2024-12-10T07:16:24,553 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6351062f9f9ac7f3c520d90dabe1dd24/A of 6351062f9f9ac7f3c520d90dabe1dd24 into ac0ff91e9a4146dba57bc0cb03931e4c(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:24,553 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:24,553 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., storeName=6351062f9f9ac7f3c520d90dabe1dd24/A, priority=13, startTime=1733814984444; duration=0sec 2024-12-10T07:16:24,553 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/d9c8bfcb059044e2935eae547fa5475a as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/d9c8bfcb059044e2935eae547fa5475a 2024-12-10T07:16:24,553 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:24,553 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6351062f9f9ac7f3c520d90dabe1dd24:A 2024-12-10T07:16:24,565 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6351062f9f9ac7f3c520d90dabe1dd24/C of 6351062f9f9ac7f3c520d90dabe1dd24 into d9c8bfcb059044e2935eae547fa5475a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:24,565 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:24,565 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., storeName=6351062f9f9ac7f3c520d90dabe1dd24/C, priority=13, startTime=1733814984445; duration=0sec 2024-12-10T07:16:24,565 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:24,565 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6351062f9f9ac7f3c520d90dabe1dd24:C 2024-12-10T07:16:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-10T07:16:24,755 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:16:24,756 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:16:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-10T07:16:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-10T07:16:24,758 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:16:24,759 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:16:24,759 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:16:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-10T07:16:24,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-10T07:16:24,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:24,917 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 6351062f9f9ac7f3c520d90dabe1dd24 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T07:16:24,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=A 2024-12-10T07:16:24,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:24,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=B 2024-12-10T07:16:24,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:24,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=C 2024-12-10T07:16:24,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:24,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412104ac2d9e8f6ea4bf0910827cdaa2a4b01_6351062f9f9ac7f3c520d90dabe1dd24 is 50, key is test_row_0/A:col10/1733814983132/Put/seqid=0 2024-12-10T07:16:24,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742248_1424 (size=12304) 2024-12-10T07:16:24,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:24,970 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412104ac2d9e8f6ea4bf0910827cdaa2a4b01_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412104ac2d9e8f6ea4bf0910827cdaa2a4b01_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:24,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/3bb6a6a7a08547448f6fc1e99b4d7952, store: [table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:24,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/3bb6a6a7a08547448f6fc1e99b4d7952 is 175, key is test_row_0/A:col10/1733814983132/Put/seqid=0 2024-12-10T07:16:24,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742249_1425 (size=31105) 2024-12-10T07:16:24,985 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/3bb6a6a7a08547448f6fc1e99b4d7952 2024-12-10T07:16:25,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/f21fb974e453494187752834a7358420 is 50, key is test_row_0/B:col10/1733814983132/Put/seqid=0 2024-12-10T07:16:25,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742250_1426 (size=12151) 2024-12-10T07:16:25,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-10T07:16:25,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:25,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:25,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:25,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815045362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:25,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:25,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:25,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815045368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:25,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-10T07:16:25,421 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/f21fb974e453494187752834a7358420 2024-12-10T07:16:25,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/9624a3378a1348da9f2a81a6a4d256f3 is 50, key is test_row_0/C:col10/1733814983132/Put/seqid=0 2024-12-10T07:16:25,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742251_1427 (size=12151) 2024-12-10T07:16:25,443 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/9624a3378a1348da9f2a81a6a4d256f3 2024-12-10T07:16:25,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/3bb6a6a7a08547448f6fc1e99b4d7952 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/3bb6a6a7a08547448f6fc1e99b4d7952 2024-12-10T07:16:25,451 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/3bb6a6a7a08547448f6fc1e99b4d7952, entries=150, sequenceid=183, filesize=30.4 K 2024-12-10T07:16:25,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/f21fb974e453494187752834a7358420 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/f21fb974e453494187752834a7358420 2024-12-10T07:16:25,457 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/f21fb974e453494187752834a7358420, entries=150, sequenceid=183, filesize=11.9 K 2024-12-10T07:16:25,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/9624a3378a1348da9f2a81a6a4d256f3 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/9624a3378a1348da9f2a81a6a4d256f3 2024-12-10T07:16:25,461 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/9624a3378a1348da9f2a81a6a4d256f3, entries=150, sequenceid=183, filesize=11.9 K 2024-12-10T07:16:25,462 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6351062f9f9ac7f3c520d90dabe1dd24 in 545ms, sequenceid=183, compaction requested=false 2024-12-10T07:16:25,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:25,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:25,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-10T07:16:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-10T07:16:25,469 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-10T07:16:25,469 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 708 msec 2024-12-10T07:16:25,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 714 msec 2024-12-10T07:16:25,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733814985354, max timestamp=1733814985478Num uniques 859; 2024-12-10T07:16:25,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733814985354, max timestamp=1733814985478Num uniques 859; 2024-12-10T07:16:25,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733814985354, max timestamp=1733814985478Num uniques 859; 2024-12-10T07:16:25,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T07:16:25,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T07:16:25,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T07:16:25,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:25,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6351062f9f9ac7f3c520d90dabe1dd24 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-12-10T07:16:25,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=A 2024-12-10T07:16:25,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:16:25,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=B 2024-12-10T07:16:25,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:16:25,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=C 2024-12-10T07:16:25,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:16:25,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100cd9709016e5440d9850459d23efafcb_6351062f9f9ac7f3c520d90dabe1dd24 is 50, key is test_row_0/A:col10/1733814985495/Put/seqid=0 2024-12-10T07:16:25,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742252_1428 (size=14794) 2024-12-10T07:16:25,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733814985518, max timestamp=1733814985588Num uniques 859; 2024-12-10T07:16:25,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733814985518, max timestamp=1733814985588Num uniques 859; 2024-12-10T07:16:25,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733814985518, max timestamp=1733814985588Num uniques 859; 2024-12-10T07:16:25,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T07:16:25,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T07:16:25,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T07:16:25,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815045697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:25,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:25,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:25,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815045699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:25,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:25,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:25,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:25,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:25,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815045804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:25,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:25,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815045814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:25,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:25,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:25,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:25,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-10T07:16:25,895 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:16:25,896 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:16:25,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-10T07:16:25,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-10T07:16:25,898 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:16:25,900 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:16:25,900 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:16:25,927 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:25,929 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412100cd9709016e5440d9850459d23efafcb_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100cd9709016e5440d9850459d23efafcb_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:25,930 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/27cc657ae38a43178aee34e37b014f08, store: [table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:25,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/27cc657ae38a43178aee34e37b014f08 is 175, key is test_row_0/A:col10/1733814985495/Put/seqid=0 2024-12-10T07:16:25,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742253_1429 (size=39749) 2024-12-10T07:16:26,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-10T07:16:26,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:26,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815046025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:26,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:26,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:26,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:26,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:26,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815046035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:26,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-10T07:16:26,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:26,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:26,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:26,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:26,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:26,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-10T07:16:26,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-10T07:16:26,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:26,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:26,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:26,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:26,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:26,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815046335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:26,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:26,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:26,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:26,356 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/27cc657ae38a43178aee34e37b014f08 2024-12-10T07:16:26,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815046356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:26,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/a71606e49f8c4e29b39955e95c3fa798 is 50, key is test_row_0/B:col10/1733814985495/Put/seqid=0 2024-12-10T07:16:26,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:26,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:26,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:26,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-10T07:16:26,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:26,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:26,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:26,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:26,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:26,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742254_1430 (size=12151) 2024-12-10T07:16:26,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/a71606e49f8c4e29b39955e95c3fa798 2024-12-10T07:16:26,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/12291eb651eb4f7f85d7f19ea1466ca9 is 50, key is test_row_0/C:col10/1733814985495/Put/seqid=0 2024-12-10T07:16:26,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742255_1431 (size=12151) 2024-12-10T07:16:26,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/12291eb651eb4f7f85d7f19ea1466ca9 2024-12-10T07:16:26,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/27cc657ae38a43178aee34e37b014f08 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/27cc657ae38a43178aee34e37b014f08 2024-12-10T07:16:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-10T07:16:26,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/27cc657ae38a43178aee34e37b014f08, entries=200, sequenceid=208, filesize=38.8 K 2024-12-10T07:16:26,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/a71606e49f8c4e29b39955e95c3fa798 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/a71606e49f8c4e29b39955e95c3fa798 2024-12-10T07:16:26,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/a71606e49f8c4e29b39955e95c3fa798, entries=150, sequenceid=208, filesize=11.9 K 2024-12-10T07:16:26,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/12291eb651eb4f7f85d7f19ea1466ca9 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/12291eb651eb4f7f85d7f19ea1466ca9 2024-12-10T07:16:26,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/12291eb651eb4f7f85d7f19ea1466ca9, entries=150, sequenceid=208, filesize=11.9 K 2024-12-10T07:16:26,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=134.18 KB/137400 for 6351062f9f9ac7f3c520d90dabe1dd24 in 1052ms, sequenceid=208, compaction requested=true 2024-12-10T07:16:26,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:26,549 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:16:26,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6351062f9f9ac7f3c520d90dabe1dd24:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:16:26,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:26,549 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:16:26,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6351062f9f9ac7f3c520d90dabe1dd24:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:16:26,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:26,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6351062f9f9ac7f3c520d90dabe1dd24:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:16:26,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:16:26,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-10T07:16:26,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:26,557 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 6351062f9f9ac7f3c520d90dabe1dd24 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-10T07:16:26,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=A 2024-12-10T07:16:26,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:16:26,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=B 2024-12-10T07:16:26,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:16:26,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=C 2024-12-10T07:16:26,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:16:26,561 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:16:26,561 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 6351062f9f9ac7f3c520d90dabe1dd24/A is initiating minor compaction (all files) 2024-12-10T07:16:26,561 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6351062f9f9ac7f3c520d90dabe1dd24/A in TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:26,561 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/ac0ff91e9a4146dba57bc0cb03931e4c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/3bb6a6a7a08547448f6fc1e99b4d7952, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/27cc657ae38a43178aee34e37b014f08] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp, totalSize=100.0 K 2024-12-10T07:16:26,561 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:26,561 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. files: [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/ac0ff91e9a4146dba57bc0cb03931e4c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/3bb6a6a7a08547448f6fc1e99b4d7952, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/27cc657ae38a43178aee34e37b014f08] 2024-12-10T07:16:26,561 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:16:26,561 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 6351062f9f9ac7f3c520d90dabe1dd24/B is initiating minor compaction (all files) 2024-12-10T07:16:26,561 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6351062f9f9ac7f3c520d90dabe1dd24/B in TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:26,561 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/39b262a38a7e4e86b809a3aa8aeab52f, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/f21fb974e453494187752834a7358420, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/a71606e49f8c4e29b39955e95c3fa798] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp, totalSize=36.0 K 2024-12-10T07:16:26,562 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 39b262a38a7e4e86b809a3aa8aeab52f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733814982743 2024-12-10T07:16:26,562 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac0ff91e9a4146dba57bc0cb03931e4c, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733814982743 2024-12-10T07:16:26,562 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting f21fb974e453494187752834a7358420, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733814983118 2024-12-10T07:16:26,562 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bb6a6a7a08547448f6fc1e99b4d7952, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733814983118 2024-12-10T07:16:26,562 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting a71606e49f8c4e29b39955e95c3fa798, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733814985478 2024-12-10T07:16:26,563 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27cc657ae38a43178aee34e37b014f08, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733814985478 2024-12-10T07:16:26,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121074371cf7693546fcbff887bd39ff396f_6351062f9f9ac7f3c520d90dabe1dd24 is 50, key is test_row_0/A:col10/1733814985685/Put/seqid=0 2024-12-10T07:16:26,599 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6351062f9f9ac7f3c520d90dabe1dd24#B#compaction#363 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:26,599 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/7f02c47747f2494881c33d5b44399bf8 is 50, key is test_row_0/B:col10/1733814985495/Put/seqid=0 2024-12-10T07:16:26,603 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:26,639 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210991dfd0159de4f11b5f69f95e10f6091_6351062f9f9ac7f3c520d90dabe1dd24 store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:26,642 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210991dfd0159de4f11b5f69f95e10f6091_6351062f9f9ac7f3c520d90dabe1dd24, store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:26,642 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210991dfd0159de4f11b5f69f95e10f6091_6351062f9f9ac7f3c520d90dabe1dd24 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:26,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742256_1432 (size=12304) 2024-12-10T07:16:26,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742258_1434 (size=4469) 2024-12-10T07:16:26,692 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6351062f9f9ac7f3c520d90dabe1dd24#A#compaction#364 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:26,693 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/075b5fd9afda42f28c2dcb0e3039aecf is 175, key is test_row_0/A:col10/1733814985495/Put/seqid=0 2024-12-10T07:16:26,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742257_1433 (size=12697) 2024-12-10T07:16:26,736 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/7f02c47747f2494881c33d5b44399bf8 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/7f02c47747f2494881c33d5b44399bf8 2024-12-10T07:16:26,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742259_1435 (size=31651) 2024-12-10T07:16:26,752 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6351062f9f9ac7f3c520d90dabe1dd24/B of 6351062f9f9ac7f3c520d90dabe1dd24 into 7f02c47747f2494881c33d5b44399bf8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:26,752 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:26,753 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., storeName=6351062f9f9ac7f3c520d90dabe1dd24/B, priority=13, startTime=1733814986549; duration=0sec 2024-12-10T07:16:26,757 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:16:26,757 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6351062f9f9ac7f3c520d90dabe1dd24:B 2024-12-10T07:16:26,757 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:16:26,758 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:16:26,758 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 6351062f9f9ac7f3c520d90dabe1dd24/C is initiating minor compaction (all files) 2024-12-10T07:16:26,758 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6351062f9f9ac7f3c520d90dabe1dd24/C in TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:26,758 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/d9c8bfcb059044e2935eae547fa5475a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/9624a3378a1348da9f2a81a6a4d256f3, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/12291eb651eb4f7f85d7f19ea1466ca9] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp, totalSize=36.0 K 2024-12-10T07:16:26,758 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting d9c8bfcb059044e2935eae547fa5475a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733814982743 2024-12-10T07:16:26,759 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 9624a3378a1348da9f2a81a6a4d256f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733814983118 2024-12-10T07:16:26,761 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 12291eb651eb4f7f85d7f19ea1466ca9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733814985478 2024-12-10T07:16:26,784 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6351062f9f9ac7f3c520d90dabe1dd24#C#compaction#365 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:26,785 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/f6a88847a34b4e81a641c05ab699f45c is 50, key is test_row_0/C:col10/1733814985495/Put/seqid=0 2024-12-10T07:16:26,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742260_1436 (size=12697) 2024-12-10T07:16:26,845 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/f6a88847a34b4e81a641c05ab699f45c as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/f6a88847a34b4e81a641c05ab699f45c 2024-12-10T07:16:26,850 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6351062f9f9ac7f3c520d90dabe1dd24/C of 6351062f9f9ac7f3c520d90dabe1dd24 into f6a88847a34b4e81a641c05ab699f45c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:26,850 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:26,850 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., storeName=6351062f9f9ac7f3c520d90dabe1dd24/C, priority=13, startTime=1733814986549; duration=0sec 2024-12-10T07:16:26,850 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:26,851 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6351062f9f9ac7f3c520d90dabe1dd24:C 2024-12-10T07:16:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:26,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:26,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815046898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:26,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:26,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:26,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:26,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815046907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:26,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:26,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:26,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:27,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815047016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:27,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:27,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:27,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:27,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815047025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:27,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:27,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:27,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-10T07:16:27,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:27,068 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121074371cf7693546fcbff887bd39ff396f_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121074371cf7693546fcbff887bd39ff396f_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:27,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/c0e84198cdea43c4b4ccf4955dd4a1f4, store: [table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:27,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/c0e84198cdea43c4b4ccf4955dd4a1f4 is 175, key is test_row_0/A:col10/1733814985685/Put/seqid=0 2024-12-10T07:16:27,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742261_1437 (size=31105) 2024-12-10T07:16:27,154 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/075b5fd9afda42f28c2dcb0e3039aecf as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/075b5fd9afda42f28c2dcb0e3039aecf 2024-12-10T07:16:27,161 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6351062f9f9ac7f3c520d90dabe1dd24/A of 6351062f9f9ac7f3c520d90dabe1dd24 into 075b5fd9afda42f28c2dcb0e3039aecf(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:27,161 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:27,161 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., storeName=6351062f9f9ac7f3c520d90dabe1dd24/A, priority=13, startTime=1733814986549; duration=0sec 2024-12-10T07:16:27,161 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:27,161 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6351062f9f9ac7f3c520d90dabe1dd24:A 2024-12-10T07:16:27,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815047225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:27,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:27,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:27,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:27,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815047235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:27,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:27,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:27,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:27,477 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/c0e84198cdea43c4b4ccf4955dd4a1f4 2024-12-10T07:16:27,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/33c33fcd31144c68926a2e3e64bd38fd is 50, key is test_row_0/B:col10/1733814985685/Put/seqid=0 2024-12-10T07:16:27,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742262_1438 (size=12151) 2024-12-10T07:16:27,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:27,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815047535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:27,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:27,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:27,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:27,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815047555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:27,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:27,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:27,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:27,888 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/33c33fcd31144c68926a2e3e64bd38fd 2024-12-10T07:16:27,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/74a534ea184c4a8b9a8c7d18cd7e00ae is 50, key is test_row_0/C:col10/1733814985685/Put/seqid=0 2024-12-10T07:16:27,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742263_1439 (size=12151) 2024-12-10T07:16:28,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815048044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:28,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:28,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:28,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-10T07:16:28,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815048065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:28,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:28,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:28,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:28,299 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/74a534ea184c4a8b9a8c7d18cd7e00ae 2024-12-10T07:16:28,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/c0e84198cdea43c4b4ccf4955dd4a1f4 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/c0e84198cdea43c4b4ccf4955dd4a1f4 2024-12-10T07:16:28,305 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/c0e84198cdea43c4b4ccf4955dd4a1f4, entries=150, sequenceid=245, filesize=30.4 K 2024-12-10T07:16:28,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/33c33fcd31144c68926a2e3e64bd38fd as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/33c33fcd31144c68926a2e3e64bd38fd 2024-12-10T07:16:28,309 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/33c33fcd31144c68926a2e3e64bd38fd, entries=150, sequenceid=245, filesize=11.9 K 2024-12-10T07:16:28,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/74a534ea184c4a8b9a8c7d18cd7e00ae as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/74a534ea184c4a8b9a8c7d18cd7e00ae 2024-12-10T07:16:28,313 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/74a534ea184c4a8b9a8c7d18cd7e00ae, entries=150, sequenceid=245, filesize=11.9 K 2024-12-10T07:16:28,314 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 6351062f9f9ac7f3c520d90dabe1dd24 in 1757ms, sequenceid=245, compaction requested=false 2024-12-10T07:16:28,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:28,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:28,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-10T07:16:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-10T07:16:28,317 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-10T07:16:28,317 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4150 sec 2024-12-10T07:16:28,318 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.4220 sec 2024-12-10T07:16:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:29,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6351062f9f9ac7f3c520d90dabe1dd24 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T07:16:29,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=A 2024-12-10T07:16:29,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:29,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=B 2024-12-10T07:16:29,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:29,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=C 2024-12-10T07:16:29,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:29,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412104906c5c27c444c47879c4208f4aaaf81_6351062f9f9ac7f3c520d90dabe1dd24 is 50, key is test_row_0/A:col10/1733814989065/Put/seqid=0 2024-12-10T07:16:29,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742264_1440 (size=14844) 2024-12-10T07:16:29,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814989115, max timestamp=1733814989137Num uniques 859; 2024-12-10T07:16:29,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814989115, max timestamp=1733814989137Num uniques 859; 2024-12-10T07:16:29,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:16:29,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:16:29,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814989115, max timestamp=1733814989137Num uniques 859; 2024-12-10T07:16:29,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:16:29,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815049183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:29,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:29,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:29,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:29,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815049184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:29,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815049294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:29,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:29,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:29,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:29,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815049304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:29,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:29,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:29,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:29,487 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:29,489 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412104906c5c27c444c47879c4208f4aaaf81_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412104906c5c27c444c47879c4208f4aaaf81_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:29,490 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/f8759e1563f54b3fb3be8332c5745c7e, store: [table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:29,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/f8759e1563f54b3fb3be8332c5745c7e is 175, key is test_row_0/A:col10/1733814989065/Put/seqid=0 2024-12-10T07:16:29,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742265_1441 (size=39799) 2024-12-10T07:16:29,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815049504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:29,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815049514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:29,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:29,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815049814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:29,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:29,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:29,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:29,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:29,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815049824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:29,893 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=259, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/f8759e1563f54b3fb3be8332c5745c7e 2024-12-10T07:16:29,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/0df5b433f463437f9eaed0c83a123c8f is 50, key is test_row_0/B:col10/1733814989065/Put/seqid=0 2024-12-10T07:16:29,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742266_1442 (size=12201) 2024-12-10T07:16:29,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/0df5b433f463437f9eaed0c83a123c8f 2024-12-10T07:16:29,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/0b6a49ea1dfe4fe0bd83696bcf2545ec is 50, key is test_row_0/C:col10/1733814989065/Put/seqid=0 2024-12-10T07:16:29,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742267_1443 (size=12201) 2024-12-10T07:16:30,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-10T07:16:30,074 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:16:30,075 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:16:30,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-10T07:16:30,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T07:16:30,076 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:16:30,077 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:16:30,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:16:30,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T07:16:30,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-10T07:16:30,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:30,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:30,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:30,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:30,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:30,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:30,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/0b6a49ea1dfe4fe0bd83696bcf2545ec 2024-12-10T07:16:30,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/f8759e1563f54b3fb3be8332c5745c7e as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/f8759e1563f54b3fb3be8332c5745c7e 2024-12-10T07:16:30,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/f8759e1563f54b3fb3be8332c5745c7e, entries=200, sequenceid=259, filesize=38.9 K 2024-12-10T07:16:30,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/0df5b433f463437f9eaed0c83a123c8f as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/0df5b433f463437f9eaed0c83a123c8f 2024-12-10T07:16:30,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/0df5b433f463437f9eaed0c83a123c8f, entries=150, sequenceid=259, filesize=11.9 K 2024-12-10T07:16:30,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/0b6a49ea1dfe4fe0bd83696bcf2545ec as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/0b6a49ea1dfe4fe0bd83696bcf2545ec 2024-12-10T07:16:30,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/0b6a49ea1dfe4fe0bd83696bcf2545ec, entries=150, sequenceid=259, filesize=11.9 K 2024-12-10T07:16:30,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 6351062f9f9ac7f3c520d90dabe1dd24 in 1265ms, sequenceid=259, compaction requested=true 2024-12-10T07:16:30,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:30,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6351062f9f9ac7f3c520d90dabe1dd24:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:16:30,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:30,331 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:16:30,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6351062f9f9ac7f3c520d90dabe1dd24:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:16:30,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:30,331 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:16:30,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6351062f9f9ac7f3c520d90dabe1dd24:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:16:30,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:16:30,332 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:16:30,332 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 6351062f9f9ac7f3c520d90dabe1dd24/B is initiating minor compaction (all files) 2024-12-10T07:16:30,332 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6351062f9f9ac7f3c520d90dabe1dd24/B in TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:30,332 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/7f02c47747f2494881c33d5b44399bf8, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/33c33fcd31144c68926a2e3e64bd38fd, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/0df5b433f463437f9eaed0c83a123c8f] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp, totalSize=36.2 K 2024-12-10T07:16:30,332 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102555 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:16:30,332 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 6351062f9f9ac7f3c520d90dabe1dd24/A is initiating minor compaction (all files) 2024-12-10T07:16:30,332 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6351062f9f9ac7f3c520d90dabe1dd24/A in TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:30,332 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/075b5fd9afda42f28c2dcb0e3039aecf, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/c0e84198cdea43c4b4ccf4955dd4a1f4, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/f8759e1563f54b3fb3be8332c5745c7e] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp, totalSize=100.2 K 2024-12-10T07:16:30,332 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f02c47747f2494881c33d5b44399bf8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733814985478 2024-12-10T07:16:30,332 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:30,332 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. files: [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/075b5fd9afda42f28c2dcb0e3039aecf, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/c0e84198cdea43c4b4ccf4955dd4a1f4, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/f8759e1563f54b3fb3be8332c5745c7e] 2024-12-10T07:16:30,333 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 33c33fcd31144c68926a2e3e64bd38fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733814985682 2024-12-10T07:16:30,333 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 075b5fd9afda42f28c2dcb0e3039aecf, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733814985478 2024-12-10T07:16:30,333 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 0df5b433f463437f9eaed0c83a123c8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733814986896 2024-12-10T07:16:30,333 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0e84198cdea43c4b4ccf4955dd4a1f4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733814985682 2024-12-10T07:16:30,333 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8759e1563f54b3fb3be8332c5745c7e, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733814986886 2024-12-10T07:16:30,337 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:30,338 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6351062f9f9ac7f3c520d90dabe1dd24#B#compaction#371 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:30,338 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/9a017ddc83dc40d990cdc2a12e328669 is 50, key is test_row_0/B:col10/1733814989065/Put/seqid=0 2024-12-10T07:16:30,341 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412108df3085203204c6a970387de88a14df5_6351062f9f9ac7f3c520d90dabe1dd24 store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:30,343 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412108df3085203204c6a970387de88a14df5_6351062f9f9ac7f3c520d90dabe1dd24, store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:30,343 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412108df3085203204c6a970387de88a14df5_6351062f9f9ac7f3c520d90dabe1dd24 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:30,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742268_1444 (size=12849) 2024-12-10T07:16:30,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:30,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6351062f9f9ac7f3c520d90dabe1dd24 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-10T07:16:30,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=A 2024-12-10T07:16:30,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:16:30,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=B 2024-12-10T07:16:30,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:16:30,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=C 2024-12-10T07:16:30,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:16:30,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742269_1445 (size=4469) 2024-12-10T07:16:30,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121072466dff407d41c4857917b286b242c3_6351062f9f9ac7f3c520d90dabe1dd24 is 50, key is test_row_0/A:col10/1733814990342/Put/seqid=0 2024-12-10T07:16:30,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742270_1446 (size=14994) 2024-12-10T07:16:30,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-10T07:16:30,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:30,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:30,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:30,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:30,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:30,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:30,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:30,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815050376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:30,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T07:16:30,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:30,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815050387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:30,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:30,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:30,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:30,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:30,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815050495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:30,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:30,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:30,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:30,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:30,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815050504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:30,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:30,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:30,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:30,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-10T07:16:30,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:30,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:30,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:30,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:30,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:30,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:30,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-10T07:16:30,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:30,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:30,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:30,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:30,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T07:16:30,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:30,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815050715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:30,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:30,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815050715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:30,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:30,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:30,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:30,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:30,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:30,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:30,749 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/9a017ddc83dc40d990cdc2a12e328669 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/9a017ddc83dc40d990cdc2a12e328669 2024-12-10T07:16:30,752 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6351062f9f9ac7f3c520d90dabe1dd24/B of 6351062f9f9ac7f3c520d90dabe1dd24 into 9a017ddc83dc40d990cdc2a12e328669(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:30,752 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:30,752 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., storeName=6351062f9f9ac7f3c520d90dabe1dd24/B, priority=13, startTime=1733814990331; duration=0sec 2024-12-10T07:16:30,752 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:16:30,752 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6351062f9f9ac7f3c520d90dabe1dd24:B 2024-12-10T07:16:30,752 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:16:30,753 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:16:30,753 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 6351062f9f9ac7f3c520d90dabe1dd24/C is initiating minor compaction (all files) 2024-12-10T07:16:30,753 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6351062f9f9ac7f3c520d90dabe1dd24/C in TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:30,753 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/f6a88847a34b4e81a641c05ab699f45c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/74a534ea184c4a8b9a8c7d18cd7e00ae, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/0b6a49ea1dfe4fe0bd83696bcf2545ec] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp, totalSize=36.2 K 2024-12-10T07:16:30,753 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting f6a88847a34b4e81a641c05ab699f45c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733814985478 2024-12-10T07:16:30,753 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 74a534ea184c4a8b9a8c7d18cd7e00ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733814985682 2024-12-10T07:16:30,753 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b6a49ea1dfe4fe0bd83696bcf2545ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733814986896 2024-12-10T07:16:30,759 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6351062f9f9ac7f3c520d90dabe1dd24#C#compaction#374 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:30,760 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/c8f0f44916cb4824925cc44ee7b7a061 is 50, key is test_row_0/C:col10/1733814989065/Put/seqid=0 2024-12-10T07:16:30,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742271_1447 (size=12849) 2024-12-10T07:16:30,763 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6351062f9f9ac7f3c520d90dabe1dd24#A#compaction#372 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:30,764 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/60542ee3e5164e678e62acded6aa5f28 is 175, key is test_row_0/A:col10/1733814989065/Put/seqid=0 2024-12-10T07:16:30,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742272_1448 (size=31803) 2024-12-10T07:16:30,779 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:30,781 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121072466dff407d41c4857917b286b242c3_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121072466dff407d41c4857917b286b242c3_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:30,782 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/37d4749ecdc641f1be2806620d0b4a60, store: [table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:30,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/37d4749ecdc641f1be2806620d0b4a60 is 175, key is test_row_0/A:col10/1733814990342/Put/seqid=0 2024-12-10T07:16:30,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742273_1449 (size=39949) 2024-12-10T07:16:30,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-10T07:16:30,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:30,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:30,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:30,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:30,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:30,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:30,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-10T07:16:30,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:30,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:30,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:30,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:30,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:31,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815051025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:31,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:31,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:31,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815051025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:31,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:31,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:31,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:31,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:31,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-10T07:16:31,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:31,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:31,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:31,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:31,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:31,166 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/c8f0f44916cb4824925cc44ee7b7a061 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/c8f0f44916cb4824925cc44ee7b7a061 2024-12-10T07:16:31,170 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6351062f9f9ac7f3c520d90dabe1dd24/C of 6351062f9f9ac7f3c520d90dabe1dd24 into c8f0f44916cb4824925cc44ee7b7a061(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:31,170 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:31,170 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., storeName=6351062f9f9ac7f3c520d90dabe1dd24/C, priority=13, startTime=1733814990331; duration=0sec 2024-12-10T07:16:31,170 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:31,170 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6351062f9f9ac7f3c520d90dabe1dd24:C 2024-12-10T07:16:31,171 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/60542ee3e5164e678e62acded6aa5f28 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/60542ee3e5164e678e62acded6aa5f28 2024-12-10T07:16:31,174 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6351062f9f9ac7f3c520d90dabe1dd24/A of 6351062f9f9ac7f3c520d90dabe1dd24 into 60542ee3e5164e678e62acded6aa5f28(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:31,174 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:31,174 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., storeName=6351062f9f9ac7f3c520d90dabe1dd24/A, priority=13, startTime=1733814990331; duration=0sec 2024-12-10T07:16:31,174 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:31,174 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6351062f9f9ac7f3c520d90dabe1dd24:A 2024-12-10T07:16:31,186 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/37d4749ecdc641f1be2806620d0b4a60 2024-12-10T07:16:31,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/646dd128b888441f808344ca17a6577d is 50, key is test_row_0/B:col10/1733814990342/Put/seqid=0 2024-12-10T07:16:31,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742274_1450 (size=12301) 2024-12-10T07:16:31,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T07:16:31,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-10T07:16:31,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:31,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:31,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:31,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:31,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:31,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-10T07:16:31,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:31,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:31,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:31,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:31,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:31,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815051544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:31,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815051545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:31,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/646dd128b888441f808344ca17a6577d 2024-12-10T07:16:31,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-10T07:16:31,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:31,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:31,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:31,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:31,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:31,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/c23b9d8bf7384fd98f6d254d26a00589 is 50, key is test_row_0/C:col10/1733814990342/Put/seqid=0 2024-12-10T07:16:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:31,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742275_1451 (size=12301) 2024-12-10T07:16:31,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/c23b9d8bf7384fd98f6d254d26a00589 2024-12-10T07:16:31,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/37d4749ecdc641f1be2806620d0b4a60 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/37d4749ecdc641f1be2806620d0b4a60 2024-12-10T07:16:31,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/37d4749ecdc641f1be2806620d0b4a60, entries=200, sequenceid=295, filesize=39.0 K 2024-12-10T07:16:31,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/646dd128b888441f808344ca17a6577d as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/646dd128b888441f808344ca17a6577d 2024-12-10T07:16:31,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/646dd128b888441f808344ca17a6577d, entries=150, sequenceid=295, filesize=12.0 K 2024-12-10T07:16:31,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/c23b9d8bf7384fd98f6d254d26a00589 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/c23b9d8bf7384fd98f6d254d26a00589 2024-12-10T07:16:31,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/c23b9d8bf7384fd98f6d254d26a00589, entries=150, sequenceid=295, filesize=12.0 K 2024-12-10T07:16:31,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 6351062f9f9ac7f3c520d90dabe1dd24 in 1295ms, sequenceid=295, compaction requested=false 2024-12-10T07:16:31,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:31,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-10T07:16:31,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:31,756 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 6351062f9f9ac7f3c520d90dabe1dd24 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T07:16:31,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=A 2024-12-10T07:16:31,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:31,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=B 2024-12-10T07:16:31,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:31,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=C 2024-12-10T07:16:31,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:31,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412105cd79a8d32b249e5b4c2055128ab02c5_6351062f9f9ac7f3c520d90dabe1dd24 is 50, key is test_row_0/A:col10/1733814990385/Put/seqid=0 2024-12-10T07:16:31,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742276_1452 (size=12454) 2024-12-10T07:16:31,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:31,770 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412105cd79a8d32b249e5b4c2055128ab02c5_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105cd79a8d32b249e5b4c2055128ab02c5_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:31,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/f30b713a6f574ff79aea8dd7298150d6, store: [table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:31,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/f30b713a6f574ff79aea8dd7298150d6 is 175, key is test_row_0/A:col10/1733814990385/Put/seqid=0 2024-12-10T07:16:31,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742277_1453 (size=31255) 2024-12-10T07:16:32,178 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=308, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/f30b713a6f574ff79aea8dd7298150d6 2024-12-10T07:16:32,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/ea0e57e49d8c43baba3e87a2cfbdcfa4 is 50, key is test_row_0/B:col10/1733814990385/Put/seqid=0 2024-12-10T07:16:32,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742278_1454 (size=12301) 2024-12-10T07:16:32,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T07:16:32,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. as already flushing 2024-12-10T07:16:32,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:32,587 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/ea0e57e49d8c43baba3e87a2cfbdcfa4 2024-12-10T07:16:32,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/12963778aecd4b0288a0001841d2cbdd is 50, key is test_row_0/C:col10/1733814990385/Put/seqid=0 2024-12-10T07:16:32,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742279_1455 (size=12301) 2024-12-10T07:16:32,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814992581, max timestamp=1733814992631Num uniques 859; 2024-12-10T07:16:32,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:16:32,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733814992581, max timestamp=1733814992631Num uniques 859; 2024-12-10T07:16:32,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-10T07:16:32,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733814992604, max timestamp=1733814992631Num uniques 859; 2024-12-10T07:16:32,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T07:16:32,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:32,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815052711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:32,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:32,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:32,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:32,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:32,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815052712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:32,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:32,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:32,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815052825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:32,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815052824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:32,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:32,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:32,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:32,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:32,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:32,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:32,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:32,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48852 deadline: 1733815052924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:32,928 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 6351062f9f9ac7f3c520d90dabe1dd24 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:32,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:32,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48848 deadline: 1733815052955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:32,955 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 6351062f9f9ac7f3c520d90dabe1dd24 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18276 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:32,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:32,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:32,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:32,997 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/12963778aecd4b0288a0001841d2cbdd 2024-12-10T07:16:33,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/f30b713a6f574ff79aea8dd7298150d6 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/f30b713a6f574ff79aea8dd7298150d6 2024-12-10T07:16:33,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:33,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48868 deadline: 1733815052994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:33,002 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 6351062f9f9ac7f3c520d90dabe1dd24 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18323 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:33,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:33,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:33,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:33,004 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/f30b713a6f574ff79aea8dd7298150d6, entries=150, sequenceid=308, filesize=30.5 K 2024-12-10T07:16:33,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/ea0e57e49d8c43baba3e87a2cfbdcfa4 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/ea0e57e49d8c43baba3e87a2cfbdcfa4 2024-12-10T07:16:33,008 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/ea0e57e49d8c43baba3e87a2cfbdcfa4, entries=150, sequenceid=308, filesize=12.0 K 2024-12-10T07:16:33,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/12963778aecd4b0288a0001841d2cbdd as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/12963778aecd4b0288a0001841d2cbdd 2024-12-10T07:16:33,011 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/12963778aecd4b0288a0001841d2cbdd, entries=150, sequenceid=308, filesize=12.0 K 2024-12-10T07:16:33,012 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for 6351062f9f9ac7f3c520d90dabe1dd24 in 1257ms, sequenceid=308, compaction requested=true 2024-12-10T07:16:33,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:33,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:33,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-10T07:16:33,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-10T07:16:33,015 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-10T07:16:33,015 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9360 sec 2024-12-10T07:16:33,016 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.9400 sec 2024-12-10T07:16:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:33,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6351062f9f9ac7f3c520d90dabe1dd24 3/3 column families, dataSize=138.65 KB heapSize=395.14 KB 2024-12-10T07:16:33,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=A 2024-12-10T07:16:33,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:16:33,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=B 2024-12-10T07:16:33,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:16:33,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=C 2024-12-10T07:16:33,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:16:33,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412106cc5b724521e4fb5a57bb90396034f57_6351062f9f9ac7f3c520d90dabe1dd24 is 50, key is test_row_0/A:col10/1733814992688/Put/seqid=0 2024-12-10T07:16:33,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742280_1456 (size=14994) 2024-12-10T07:16:33,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:33,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815053069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:33,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815053070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:33,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:33,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:33,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:33,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:33,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815053175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:33,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:33,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:33,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:33,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:33,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815053184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:33,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:33,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815053385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:33,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:33,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:33,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:33,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:33,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815053395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:33,444 INFO [Thread-1675 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1675. 2024-12-10T07:16:33,444 DEBUG [Thread-1675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:16:33,444 DEBUG [Thread-1675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:33,444 DEBUG [Thread-1675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:33,444 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:16:33,445 INFO [Thread-1669 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1669. 2024-12-10T07:16:33,445 DEBUG [Thread-1669 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:16:33,445 DEBUG [Thread-1669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:33,445 DEBUG [Thread-1669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:33,445 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:16:33,447 INFO [Thread-1667 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1667. 2024-12-10T07:16:33,447 DEBUG [Thread-1667 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:16:33,447 DEBUG [Thread-1667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:33,447 DEBUG [Thread-1667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:33,447 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:16:33,450 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:33,453 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412106cc5b724521e4fb5a57bb90396034f57_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412106cc5b724521e4fb5a57bb90396034f57_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:33,453 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/9b2a9f93383e4259adeec40f49f6a717, store: [table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:33,453 INFO [Thread-1671 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1671. 2024-12-10T07:16:33,454 DEBUG [Thread-1671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:16:33,454 DEBUG [Thread-1671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:33,454 DEBUG [Thread-1671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:33,454 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:16:33,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/9b2a9f93383e4259adeec40f49f6a717 is 175, key is test_row_0/A:col10/1733814992688/Put/seqid=0 2024-12-10T07:16:33,455 INFO [Thread-1673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1673. 2024-12-10T07:16:33,455 DEBUG [Thread-1673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:16:33,455 DEBUG [Thread-1673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:33,455 DEBUG [Thread-1673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:33,455 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:16:33,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742281_1457 (size=39949) 2024-12-10T07:16:33,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:33,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815053695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:33,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:33,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:33,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:33,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:33,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815053704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:33,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:33,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:33,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:33,710 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T07:16:33,859 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=345, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/9b2a9f93383e4259adeec40f49f6a717 2024-12-10T07:16:33,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/d87a5451028d4dfd86634f94fef490e0 is 50, key is test_row_0/B:col10/1733814992688/Put/seqid=0 2024-12-10T07:16:33,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742282_1458 (size=12301) 2024-12-10T07:16:34,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:34,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48834 deadline: 1733815054205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:34,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:34,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:34,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:34,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:34,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48866 deadline: 1733815054215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:34,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:34,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:34,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:16:34,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-10T07:16:34,245 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:16:34,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/d87a5451028d4dfd86634f94fef490e0 2024-12-10T07:16:34,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/7b62ec40df3c4a17a9a9a531818a8a7f is 50, key is test_row_0/C:col10/1733814992688/Put/seqid=0 2024-12-10T07:16:34,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742283_1459 (size=12301) 2024-12-10T07:16:34,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/7b62ec40df3c4a17a9a9a531818a8a7f 2024-12-10T07:16:34,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/9b2a9f93383e4259adeec40f49f6a717 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/9b2a9f93383e4259adeec40f49f6a717 2024-12-10T07:16:34,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/9b2a9f93383e4259adeec40f49f6a717, entries=200, sequenceid=345, filesize=39.0 K 2024-12-10T07:16:34,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/d87a5451028d4dfd86634f94fef490e0 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/d87a5451028d4dfd86634f94fef490e0 2024-12-10T07:16:34,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/d87a5451028d4dfd86634f94fef490e0, entries=150, sequenceid=345, filesize=12.0 K 2024-12-10T07:16:34,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/7b62ec40df3c4a17a9a9a531818a8a7f as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/7b62ec40df3c4a17a9a9a531818a8a7f 2024-12-10T07:16:34,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/7b62ec40df3c4a17a9a9a531818a8a7f, entries=150, sequenceid=345, filesize=12.0 K 2024-12-10T07:16:34,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~395.09 KB/404576, currentSize=40.25 KB/41220 for 6351062f9f9ac7f3c520d90dabe1dd24 in 1678ms, sequenceid=345, compaction requested=true 2024-12-10T07:16:34,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:34,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6351062f9f9ac7f3c520d90dabe1dd24:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:16:34,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:34,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6351062f9f9ac7f3c520d90dabe1dd24:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:16:34,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:34,717 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:16:34,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6351062f9f9ac7f3c520d90dabe1dd24:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:16:34,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:16:34,717 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:16:34,718 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49752 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:16:34,718 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142956 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:16:34,718 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 6351062f9f9ac7f3c520d90dabe1dd24/A is initiating minor compaction (all files) 2024-12-10T07:16:34,718 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 6351062f9f9ac7f3c520d90dabe1dd24/B is initiating minor compaction (all files) 2024-12-10T07:16:34,718 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6351062f9f9ac7f3c520d90dabe1dd24/A in TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:34,718 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6351062f9f9ac7f3c520d90dabe1dd24/B in TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:34,718 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/60542ee3e5164e678e62acded6aa5f28, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/37d4749ecdc641f1be2806620d0b4a60, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/f30b713a6f574ff79aea8dd7298150d6, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/9b2a9f93383e4259adeec40f49f6a717] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp, totalSize=139.6 K 2024-12-10T07:16:34,718 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/9a017ddc83dc40d990cdc2a12e328669, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/646dd128b888441f808344ca17a6577d, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/ea0e57e49d8c43baba3e87a2cfbdcfa4, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/d87a5451028d4dfd86634f94fef490e0] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp, totalSize=48.6 K 2024-12-10T07:16:34,719 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:34,719 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. files: [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/60542ee3e5164e678e62acded6aa5f28, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/37d4749ecdc641f1be2806620d0b4a60, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/f30b713a6f574ff79aea8dd7298150d6, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/9b2a9f93383e4259adeec40f49f6a717] 2024-12-10T07:16:34,719 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a017ddc83dc40d990cdc2a12e328669, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733814986896 2024-12-10T07:16:34,719 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60542ee3e5164e678e62acded6aa5f28, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733814986896 2024-12-10T07:16:34,719 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 646dd128b888441f808344ca17a6577d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733814989182 2024-12-10T07:16:34,719 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37d4749ecdc641f1be2806620d0b4a60, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733814989166 2024-12-10T07:16:34,719 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting ea0e57e49d8c43baba3e87a2cfbdcfa4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733814990373 2024-12-10T07:16:34,719 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting f30b713a6f574ff79aea8dd7298150d6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733814990373 2024-12-10T07:16:34,719 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting d87a5451028d4dfd86634f94fef490e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733814992688 2024-12-10T07:16:34,719 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b2a9f93383e4259adeec40f49f6a717, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733814992688 2024-12-10T07:16:34,724 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:34,725 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6351062f9f9ac7f3c520d90dabe1dd24#B#compaction#383 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:34,726 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/0ef8ad8682724903b2673483b2617529 is 50, key is test_row_0/B:col10/1733814992688/Put/seqid=0 2024-12-10T07:16:34,726 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210415058f9353d42a28c83dccd3b1e0437_6351062f9f9ac7f3c520d90dabe1dd24 store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:34,730 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210415058f9353d42a28c83dccd3b1e0437_6351062f9f9ac7f3c520d90dabe1dd24, store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:34,730 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210415058f9353d42a28c83dccd3b1e0437_6351062f9f9ac7f3c520d90dabe1dd24 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:34,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742284_1460 (size=13085) 2024-12-10T07:16:34,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742285_1461 (size=4469) 2024-12-10T07:16:35,134 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6351062f9f9ac7f3c520d90dabe1dd24#A#compaction#384 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:35,136 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/b3fee7d05d5d48cfbab022dcd3a53144 is 175, key is test_row_0/A:col10/1733814992688/Put/seqid=0 2024-12-10T07:16:35,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742286_1462 (size=32039) 2024-12-10T07:16:35,143 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/0ef8ad8682724903b2673483b2617529 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/0ef8ad8682724903b2673483b2617529 2024-12-10T07:16:35,146 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6351062f9f9ac7f3c520d90dabe1dd24/B of 6351062f9f9ac7f3c520d90dabe1dd24 into 0ef8ad8682724903b2673483b2617529(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:35,146 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:35,146 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., storeName=6351062f9f9ac7f3c520d90dabe1dd24/B, priority=12, startTime=1733814994717; duration=0sec 2024-12-10T07:16:35,147 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:16:35,147 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6351062f9f9ac7f3c520d90dabe1dd24:B 2024-12-10T07:16:35,147 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:16:35,147 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49752 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:16:35,147 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 6351062f9f9ac7f3c520d90dabe1dd24/C is initiating minor compaction (all files) 2024-12-10T07:16:35,147 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6351062f9f9ac7f3c520d90dabe1dd24/C in TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:35,148 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/c8f0f44916cb4824925cc44ee7b7a061, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/c23b9d8bf7384fd98f6d254d26a00589, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/12963778aecd4b0288a0001841d2cbdd, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/7b62ec40df3c4a17a9a9a531818a8a7f] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp, totalSize=48.6 K 2024-12-10T07:16:35,148 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting c8f0f44916cb4824925cc44ee7b7a061, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733814986896 2024-12-10T07:16:35,148 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting c23b9d8bf7384fd98f6d254d26a00589, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733814989182 2024-12-10T07:16:35,148 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 12963778aecd4b0288a0001841d2cbdd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733814990373 2024-12-10T07:16:35,148 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b62ec40df3c4a17a9a9a531818a8a7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733814992688 2024-12-10T07:16:35,153 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6351062f9f9ac7f3c520d90dabe1dd24#C#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:16:35,154 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/262c0c2bc180405199ccb476b5350117 is 50, key is test_row_0/C:col10/1733814992688/Put/seqid=0 2024-12-10T07:16:35,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742287_1463 (size=13085) 2024-12-10T07:16:35,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:35,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6351062f9f9ac7f3c520d90dabe1dd24 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-10T07:16:35,226 INFO [Thread-1662 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1662. 2024-12-10T07:16:35,226 DEBUG [Thread-1662 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:16:35,226 INFO [Thread-1658 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1658. 2024-12-10T07:16:35,226 DEBUG [Thread-1662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:35,226 DEBUG [Thread-1658 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:16:35,226 DEBUG [Thread-1658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:35,226 DEBUG [Thread-1662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:35,226 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:16:35,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=A 2024-12-10T07:16:35,226 DEBUG [Thread-1658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:35,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:35,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=B 2024-12-10T07:16:35,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:35,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=C 2024-12-10T07:16:35,226 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:16:35,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:35,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210ab3b89cec8f34384a4251c9cd2be34c9_6351062f9f9ac7f3c520d90dabe1dd24 is 50, key is test_row_0/A:col10/1733814995225/Put/seqid=0 2024-12-10T07:16:35,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742288_1464 (size=12454) 2024-12-10T07:16:35,554 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/b3fee7d05d5d48cfbab022dcd3a53144 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/b3fee7d05d5d48cfbab022dcd3a53144 2024-12-10T07:16:35,560 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6351062f9f9ac7f3c520d90dabe1dd24/A of 6351062f9f9ac7f3c520d90dabe1dd24 into b3fee7d05d5d48cfbab022dcd3a53144(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:35,560 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:35,560 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., storeName=6351062f9f9ac7f3c520d90dabe1dd24/A, priority=12, startTime=1733814994717; duration=0sec 2024-12-10T07:16:35,561 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:35,561 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6351062f9f9ac7f3c520d90dabe1dd24:A 2024-12-10T07:16:35,561 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/262c0c2bc180405199ccb476b5350117 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/262c0c2bc180405199ccb476b5350117 2024-12-10T07:16:35,566 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6351062f9f9ac7f3c520d90dabe1dd24/C of 6351062f9f9ac7f3c520d90dabe1dd24 into 262c0c2bc180405199ccb476b5350117(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:16:35,566 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:35,566 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., storeName=6351062f9f9ac7f3c520d90dabe1dd24/C, priority=12, startTime=1733814994717; duration=0sec 2024-12-10T07:16:35,566 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:16:35,566 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6351062f9f9ac7f3c520d90dabe1dd24:C 2024-12-10T07:16:35,634 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:35,640 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210ab3b89cec8f34384a4251c9cd2be34c9_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ab3b89cec8f34384a4251c9cd2be34c9_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:35,641 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/12633e7d44324e40935ca9f273ea9660, store: [table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:35,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/12633e7d44324e40935ca9f273ea9660 is 175, key is test_row_0/A:col10/1733814995225/Put/seqid=0 2024-12-10T07:16:35,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742289_1465 (size=31255) 2024-12-10T07:16:36,047 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=357, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/12633e7d44324e40935ca9f273ea9660 2024-12-10T07:16:36,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/56b198c0eb2441a2893385684e3eefaa is 50, key is test_row_0/B:col10/1733814995225/Put/seqid=0 2024-12-10T07:16:36,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742290_1466 (size=12301) 2024-12-10T07:16:36,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/56b198c0eb2441a2893385684e3eefaa 2024-12-10T07:16:36,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/bbe04c6c1e01406bbccfac85e1adac42 is 50, key is test_row_0/C:col10/1733814995225/Put/seqid=0 2024-12-10T07:16:36,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742291_1467 (size=12301) 2024-12-10T07:16:36,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/bbe04c6c1e01406bbccfac85e1adac42 2024-12-10T07:16:36,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/12633e7d44324e40935ca9f273ea9660 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/12633e7d44324e40935ca9f273ea9660 2024-12-10T07:16:36,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/12633e7d44324e40935ca9f273ea9660, entries=150, sequenceid=357, filesize=30.5 K 2024-12-10T07:16:36,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/56b198c0eb2441a2893385684e3eefaa as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/56b198c0eb2441a2893385684e3eefaa 2024-12-10T07:16:36,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/56b198c0eb2441a2893385684e3eefaa, entries=150, sequenceid=357, filesize=12.0 K 2024-12-10T07:16:36,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/bbe04c6c1e01406bbccfac85e1adac42 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/bbe04c6c1e01406bbccfac85e1adac42 2024-12-10T07:16:36,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/bbe04c6c1e01406bbccfac85e1adac42, entries=150, sequenceid=357, filesize=12.0 K 2024-12-10T07:16:36,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for 6351062f9f9ac7f3c520d90dabe1dd24 in 1674ms, sequenceid=357, compaction requested=false 2024-12-10T07:16:36,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6351062f9f9ac7f3c520d90dabe1dd24: 2024-12-10T07:16:38,245 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b1396b8d2d12:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/8e7f2e03ca7747caa15ed01ce321c6e7, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/85a4b232bada44e694b37191a46759ff, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/58309db390694cd2901f6df07a2a5646, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/4ecfcf9aa5184aa9a0bffd4f06cd4d2d, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/63b861846f1a438f9ce1eba8cba99ab2, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/26eff80184bc4015957edd1deeb3315c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/a068f16540a840c29c2a43ae0bf2c812, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/7dea415dce57440c94f1df77887b8f64, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/92f90ef305ae42a0ae9bab37e6735ce2, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/af4361d0cd1847a2aafabd242b2fce0e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/c9debd96edd2449c9306576198b6773d, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/853728224f414732af03b746fe1ff4ba, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/97c0be489cd649e2b26e18c7d8605f67, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/ac0ff91e9a4146dba57bc0cb03931e4c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/3bb6a6a7a08547448f6fc1e99b4d7952, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/27cc657ae38a43178aee34e37b014f08, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/075b5fd9afda42f28c2dcb0e3039aecf, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/c0e84198cdea43c4b4ccf4955dd4a1f4, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/f8759e1563f54b3fb3be8332c5745c7e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/60542ee3e5164e678e62acded6aa5f28, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/37d4749ecdc641f1be2806620d0b4a60, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/f30b713a6f574ff79aea8dd7298150d6, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/9b2a9f93383e4259adeec40f49f6a717] to archive 2024-12-10T07:16:38,246 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b1396b8d2d12:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T07:16:38,248 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/85a4b232bada44e694b37191a46759ff to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/85a4b232bada44e694b37191a46759ff 2024-12-10T07:16:38,248 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/8e7f2e03ca7747caa15ed01ce321c6e7 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/8e7f2e03ca7747caa15ed01ce321c6e7 2024-12-10T07:16:38,248 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/58309db390694cd2901f6df07a2a5646 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/58309db390694cd2901f6df07a2a5646 2024-12-10T07:16:38,248 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/63b861846f1a438f9ce1eba8cba99ab2 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/63b861846f1a438f9ce1eba8cba99ab2 2024-12-10T07:16:38,248 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/4ecfcf9aa5184aa9a0bffd4f06cd4d2d to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/4ecfcf9aa5184aa9a0bffd4f06cd4d2d 2024-12-10T07:16:38,248 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/a068f16540a840c29c2a43ae0bf2c812 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/a068f16540a840c29c2a43ae0bf2c812 2024-12-10T07:16:38,248 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/7dea415dce57440c94f1df77887b8f64 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/7dea415dce57440c94f1df77887b8f64 2024-12-10T07:16:38,249 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/26eff80184bc4015957edd1deeb3315c to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/26eff80184bc4015957edd1deeb3315c 2024-12-10T07:16:38,250 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/92f90ef305ae42a0ae9bab37e6735ce2 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/92f90ef305ae42a0ae9bab37e6735ce2 2024-12-10T07:16:38,250 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/853728224f414732af03b746fe1ff4ba to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/853728224f414732af03b746fe1ff4ba 2024-12-10T07:16:38,250 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/af4361d0cd1847a2aafabd242b2fce0e to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/af4361d0cd1847a2aafabd242b2fce0e 2024-12-10T07:16:38,250 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/3bb6a6a7a08547448f6fc1e99b4d7952 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/3bb6a6a7a08547448f6fc1e99b4d7952 2024-12-10T07:16:38,250 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/ac0ff91e9a4146dba57bc0cb03931e4c to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/ac0ff91e9a4146dba57bc0cb03931e4c 2024-12-10T07:16:38,250 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/c9debd96edd2449c9306576198b6773d to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/c9debd96edd2449c9306576198b6773d 2024-12-10T07:16:38,250 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/97c0be489cd649e2b26e18c7d8605f67 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/97c0be489cd649e2b26e18c7d8605f67 2024-12-10T07:16:38,250 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/27cc657ae38a43178aee34e37b014f08 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/27cc657ae38a43178aee34e37b014f08 2024-12-10T07:16:38,251 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/075b5fd9afda42f28c2dcb0e3039aecf to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/075b5fd9afda42f28c2dcb0e3039aecf 2024-12-10T07:16:38,251 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/c0e84198cdea43c4b4ccf4955dd4a1f4 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/c0e84198cdea43c4b4ccf4955dd4a1f4 2024-12-10T07:16:38,251 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/f8759e1563f54b3fb3be8332c5745c7e to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/f8759e1563f54b3fb3be8332c5745c7e 2024-12-10T07:16:38,251 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/60542ee3e5164e678e62acded6aa5f28 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/60542ee3e5164e678e62acded6aa5f28 2024-12-10T07:16:38,252 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/f30b713a6f574ff79aea8dd7298150d6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/f30b713a6f574ff79aea8dd7298150d6 2024-12-10T07:16:38,252 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/9b2a9f93383e4259adeec40f49f6a717 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/9b2a9f93383e4259adeec40f49f6a717 2024-12-10T07:16:38,252 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/37d4749ecdc641f1be2806620d0b4a60 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/37d4749ecdc641f1be2806620d0b4a60 2024-12-10T07:16:38,255 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b1396b8d2d12:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/0d1a2c5696944b20a2983817099eaff6, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/5d642eaf950444b58ddd9c6b0151ade1, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/1afbe93976f947da8cbe8fdcb5aca701, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/c8722539973e4fc9a4de73c26283006e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/100bc14414684003ab8acf377c2039e6, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/952d41684fe642b5bc4860effde55e89, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/de31edcef07841caaf3656d984960a27, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/f0717babdb8448e2a552bbd866694a29, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/ba95a51b17c248ecabb971f2a1b7be46, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/835024302ea74862bc2be89944d77303, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/b171910e6ccf43fdb5e05a9605e9880f, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/6743b16aa8bd48e2afd7b1543c3ce981, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/39b262a38a7e4e86b809a3aa8aeab52f, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/0cada39f52e6477e99c3a773651f212d, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/f21fb974e453494187752834a7358420, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/7f02c47747f2494881c33d5b44399bf8, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/a71606e49f8c4e29b39955e95c3fa798, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/33c33fcd31144c68926a2e3e64bd38fd, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/9a017ddc83dc40d990cdc2a12e328669, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/0df5b433f463437f9eaed0c83a123c8f, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/646dd128b888441f808344ca17a6577d, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/ea0e57e49d8c43baba3e87a2cfbdcfa4, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/d87a5451028d4dfd86634f94fef490e0] to archive 2024-12-10T07:16:38,256 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b1396b8d2d12:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T07:16:38,259 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/1afbe93976f947da8cbe8fdcb5aca701 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/1afbe93976f947da8cbe8fdcb5aca701 2024-12-10T07:16:38,259 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/0d1a2c5696944b20a2983817099eaff6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/0d1a2c5696944b20a2983817099eaff6 2024-12-10T07:16:38,259 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/5d642eaf950444b58ddd9c6b0151ade1 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/5d642eaf950444b58ddd9c6b0151ade1 2024-12-10T07:16:38,259 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/c8722539973e4fc9a4de73c26283006e to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/c8722539973e4fc9a4de73c26283006e 2024-12-10T07:16:38,260 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/100bc14414684003ab8acf377c2039e6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/100bc14414684003ab8acf377c2039e6 2024-12-10T07:16:38,260 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/f0717babdb8448e2a552bbd866694a29 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/f0717babdb8448e2a552bbd866694a29 2024-12-10T07:16:38,260 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/952d41684fe642b5bc4860effde55e89 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/952d41684fe642b5bc4860effde55e89 2024-12-10T07:16:38,260 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/de31edcef07841caaf3656d984960a27 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/de31edcef07841caaf3656d984960a27 2024-12-10T07:16:38,261 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/ba95a51b17c248ecabb971f2a1b7be46 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/ba95a51b17c248ecabb971f2a1b7be46 2024-12-10T07:16:38,261 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/835024302ea74862bc2be89944d77303 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/835024302ea74862bc2be89944d77303 2024-12-10T07:16:38,261 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/b171910e6ccf43fdb5e05a9605e9880f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/b171910e6ccf43fdb5e05a9605e9880f 2024-12-10T07:16:38,262 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/6743b16aa8bd48e2afd7b1543c3ce981 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/6743b16aa8bd48e2afd7b1543c3ce981 2024-12-10T07:16:38,262 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/39b262a38a7e4e86b809a3aa8aeab52f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/39b262a38a7e4e86b809a3aa8aeab52f 2024-12-10T07:16:38,262 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/f21fb974e453494187752834a7358420 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/f21fb974e453494187752834a7358420 2024-12-10T07:16:38,262 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/0cada39f52e6477e99c3a773651f212d to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/0cada39f52e6477e99c3a773651f212d 2024-12-10T07:16:38,262 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/7f02c47747f2494881c33d5b44399bf8 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/7f02c47747f2494881c33d5b44399bf8 2024-12-10T07:16:38,262 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/33c33fcd31144c68926a2e3e64bd38fd to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/33c33fcd31144c68926a2e3e64bd38fd 2024-12-10T07:16:38,263 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/a71606e49f8c4e29b39955e95c3fa798 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/a71606e49f8c4e29b39955e95c3fa798 2024-12-10T07:16:38,263 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/9a017ddc83dc40d990cdc2a12e328669 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/9a017ddc83dc40d990cdc2a12e328669 2024-12-10T07:16:38,263 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/0df5b433f463437f9eaed0c83a123c8f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/0df5b433f463437f9eaed0c83a123c8f 2024-12-10T07:16:38,263 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/646dd128b888441f808344ca17a6577d to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/646dd128b888441f808344ca17a6577d 2024-12-10T07:16:38,263 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/ea0e57e49d8c43baba3e87a2cfbdcfa4 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/ea0e57e49d8c43baba3e87a2cfbdcfa4 2024-12-10T07:16:38,263 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/d87a5451028d4dfd86634f94fef490e0 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/d87a5451028d4dfd86634f94fef490e0 2024-12-10T07:16:38,266 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b1396b8d2d12:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/2e9704620a554ed48e96bbac8755f9ba, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/50017646c67f44e7ae6ae34e655ae7bd, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/005f222920cf4d5da31284acd9af52a7, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/d82b17b294ed4f5d81b80c1ab7e5b15d, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/706ef29a0a12425daca0b004ddeadb6e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/042c2bc8f6c94ce688a0f1a931a1a0d7, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/36e0d50a233d49609acd1653f48a9b73, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/c355d924c3d449d29d4c9367786ca88b, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/68c2f5882e394796a087e3d30a662b54, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/c69253f3e0ba46418457da6cc30e0a0b, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/e660ba6182f1486fbee19f9f79e683b4, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/bfb25d07914643ec9438cfc379183521, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/d9c8bfcb059044e2935eae547fa5475a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/2e3710bb5ae6469f8169617a4dcaa419, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/9624a3378a1348da9f2a81a6a4d256f3, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/f6a88847a34b4e81a641c05ab699f45c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/12291eb651eb4f7f85d7f19ea1466ca9, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/74a534ea184c4a8b9a8c7d18cd7e00ae, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/c8f0f44916cb4824925cc44ee7b7a061, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/0b6a49ea1dfe4fe0bd83696bcf2545ec, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/c23b9d8bf7384fd98f6d254d26a00589, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/12963778aecd4b0288a0001841d2cbdd, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/7b62ec40df3c4a17a9a9a531818a8a7f] to archive 2024-12-10T07:16:38,267 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/b1396b8d2d12:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T07:16:38,269 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/2e9704620a554ed48e96bbac8755f9ba to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/2e9704620a554ed48e96bbac8755f9ba 2024-12-10T07:16:38,269 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/d82b17b294ed4f5d81b80c1ab7e5b15d to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/d82b17b294ed4f5d81b80c1ab7e5b15d 2024-12-10T07:16:38,269 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/005f222920cf4d5da31284acd9af52a7 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/005f222920cf4d5da31284acd9af52a7 2024-12-10T07:16:38,269 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/706ef29a0a12425daca0b004ddeadb6e to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/706ef29a0a12425daca0b004ddeadb6e 2024-12-10T07:16:38,269 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/36e0d50a233d49609acd1653f48a9b73 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/36e0d50a233d49609acd1653f48a9b73 2024-12-10T07:16:38,269 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/042c2bc8f6c94ce688a0f1a931a1a0d7 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/042c2bc8f6c94ce688a0f1a931a1a0d7 2024-12-10T07:16:38,269 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/c355d924c3d449d29d4c9367786ca88b to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/c355d924c3d449d29d4c9367786ca88b 2024-12-10T07:16:38,269 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/50017646c67f44e7ae6ae34e655ae7bd to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/50017646c67f44e7ae6ae34e655ae7bd 2024-12-10T07:16:38,270 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/68c2f5882e394796a087e3d30a662b54 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/68c2f5882e394796a087e3d30a662b54 2024-12-10T07:16:38,270 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/e660ba6182f1486fbee19f9f79e683b4 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/e660ba6182f1486fbee19f9f79e683b4 2024-12-10T07:16:38,270 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/2e3710bb5ae6469f8169617a4dcaa419 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/2e3710bb5ae6469f8169617a4dcaa419 2024-12-10T07:16:38,270 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/bfb25d07914643ec9438cfc379183521 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/bfb25d07914643ec9438cfc379183521 2024-12-10T07:16:38,270 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/d9c8bfcb059044e2935eae547fa5475a to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/d9c8bfcb059044e2935eae547fa5475a 2024-12-10T07:16:38,270 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/c69253f3e0ba46418457da6cc30e0a0b to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/c69253f3e0ba46418457da6cc30e0a0b 2024-12-10T07:16:38,270 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/9624a3378a1348da9f2a81a6a4d256f3 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/9624a3378a1348da9f2a81a6a4d256f3 2024-12-10T07:16:38,271 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/f6a88847a34b4e81a641c05ab699f45c to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/f6a88847a34b4e81a641c05ab699f45c 2024-12-10T07:16:38,271 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/12291eb651eb4f7f85d7f19ea1466ca9 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/12291eb651eb4f7f85d7f19ea1466ca9 2024-12-10T07:16:38,271 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/74a534ea184c4a8b9a8c7d18cd7e00ae to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/74a534ea184c4a8b9a8c7d18cd7e00ae 2024-12-10T07:16:38,271 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/0b6a49ea1dfe4fe0bd83696bcf2545ec to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/0b6a49ea1dfe4fe0bd83696bcf2545ec 2024-12-10T07:16:38,272 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/c23b9d8bf7384fd98f6d254d26a00589 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/c23b9d8bf7384fd98f6d254d26a00589 2024-12-10T07:16:38,272 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/12963778aecd4b0288a0001841d2cbdd to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/12963778aecd4b0288a0001841d2cbdd 2024-12-10T07:16:38,272 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/7b62ec40df3c4a17a9a9a531818a8a7f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/7b62ec40df3c4a17a9a9a531818a8a7f 2024-12-10T07:16:38,272 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/c8f0f44916cb4824925cc44ee7b7a061 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/c8f0f44916cb4824925cc44ee7b7a061 2024-12-10T07:16:42,998 INFO [Thread-1656 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1656. 2024-12-10T07:16:42,999 DEBUG [Thread-1656 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:16:42,999 DEBUG [Thread-1656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:42,999 DEBUG [Thread-1656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:42,999 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:16:43,009 INFO [Thread-1660 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1660. 2024-12-10T07:16:43,009 DEBUG [Thread-1660 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:16:43,009 DEBUG [Thread-1660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:43,009 DEBUG [Thread-1660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:43,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:16:43,099 INFO [Thread-1664 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1664. 2024-12-10T07:16:43,099 DEBUG [Thread-1664 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:16:43,099 DEBUG [Thread-1664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:43,099 DEBUG [Thread-1664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:43,099 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:16:43,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T07:16:43,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-12-10T07:16:43,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 125 2024-12-10T07:16:43,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-12-10T07:16:43,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 130 2024-12-10T07:16:43,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-12-10T07:16:43,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T07:16:43,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T07:16:43,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1359 2024-12-10T07:16:43,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4064 rows 2024-12-10T07:16:43,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1331 2024-12-10T07:16:43,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3993 rows 2024-12-10T07:16:43,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1335 2024-12-10T07:16:43,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4000 rows 2024-12-10T07:16:43,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1329 2024-12-10T07:16:43,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3987 rows 2024-12-10T07:16:43,100 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1345 2024-12-10T07:16:43,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4029 rows 2024-12-10T07:16:43,101 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T07:16:43,101 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T07:16:43,101 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:43,107 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:43,107 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:16:43,108 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-10T07:16:43,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T07:16:43,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-10T07:16:43,110 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733815003110"}]},"ts":"1733815003110"} 2024-12-10T07:16:43,111 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T07:16:43,111 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T07:16:43,111 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T07:16:43,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6351062f9f9ac7f3c520d90dabe1dd24, UNASSIGN}] 2024-12-10T07:16:43,113 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6351062f9f9ac7f3c520d90dabe1dd24, UNASSIGN 2024-12-10T07:16:43,113 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=6351062f9f9ac7f3c520d90dabe1dd24, regionState=CLOSING, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:43,114 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6351062f9f9ac7f3c520d90dabe1dd24, UNASSIGN because future has completed 2024-12-10T07:16:43,115 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T07:16:43,115 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835}] 2024-12-10T07:16:43,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-10T07:16:43,267 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:43,267 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T07:16:43,267 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing 6351062f9f9ac7f3c520d90dabe1dd24, disabling compactions & flushes 2024-12-10T07:16:43,267 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:43,267 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:43,267 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. after waiting 0 ms 2024-12-10T07:16:43,267 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:43,268 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(2902): Flushing 6351062f9f9ac7f3c520d90dabe1dd24 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-10T07:16:43,268 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=A 2024-12-10T07:16:43,268 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:43,269 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=B 2024-12-10T07:16:43,269 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:43,269 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6351062f9f9ac7f3c520d90dabe1dd24, store=C 2024-12-10T07:16:43,269 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:43,279 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121091e04e6a031649ffa03dff999cae5736_6351062f9f9ac7f3c520d90dabe1dd24 is 50, key is test_row_1/A:col10/1733815003096/Put/seqid=0 2024-12-10T07:16:43,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742292_1468 (size=7374) 2024-12-10T07:16:43,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-10T07:16:43,684 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:43,692 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121091e04e6a031649ffa03dff999cae5736_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121091e04e6a031649ffa03dff999cae5736_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:43,693 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/75ec5d8e77fb403d82e4a69c3375fd7c, store: [table=TestAcidGuarantees family=A region=6351062f9f9ac7f3c520d90dabe1dd24] 2024-12-10T07:16:43,694 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/75ec5d8e77fb403d82e4a69c3375fd7c is 175, key is test_row_1/A:col10/1733815003096/Put/seqid=0 2024-12-10T07:16:43,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742293_1469 (size=13865) 2024-12-10T07:16:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-10T07:16:44,101 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=365, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/75ec5d8e77fb403d82e4a69c3375fd7c 2024-12-10T07:16:44,113 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/4b1a257e8a1c413783a7ba4c41573731 is 50, key is test_row_1/B:col10/1733815003096/Put/seqid=0 2024-12-10T07:16:44,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742294_1470 (size=7415) 2024-12-10T07:16:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-10T07:16:44,518 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/4b1a257e8a1c413783a7ba4c41573731 2024-12-10T07:16:44,529 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/63ebcf3b28f04fe2994dd532713e41df is 50, key is test_row_1/C:col10/1733815003096/Put/seqid=0 2024-12-10T07:16:44,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742295_1471 (size=7415) 2024-12-10T07:16:44,933 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/63ebcf3b28f04fe2994dd532713e41df 2024-12-10T07:16:44,942 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/A/75ec5d8e77fb403d82e4a69c3375fd7c as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/75ec5d8e77fb403d82e4a69c3375fd7c 2024-12-10T07:16:44,947 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/75ec5d8e77fb403d82e4a69c3375fd7c, entries=50, sequenceid=365, filesize=13.5 K 2024-12-10T07:16:44,948 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/B/4b1a257e8a1c413783a7ba4c41573731 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/4b1a257e8a1c413783a7ba4c41573731 2024-12-10T07:16:44,951 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/4b1a257e8a1c413783a7ba4c41573731, entries=50, sequenceid=365, filesize=7.2 K 2024-12-10T07:16:44,951 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/.tmp/C/63ebcf3b28f04fe2994dd532713e41df as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/63ebcf3b28f04fe2994dd532713e41df 2024-12-10T07:16:44,954 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/63ebcf3b28f04fe2994dd532713e41df, entries=50, sequenceid=365, filesize=7.2 K 2024-12-10T07:16:44,955 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 6351062f9f9ac7f3c520d90dabe1dd24 in 1688ms, sequenceid=365, compaction requested=true 2024-12-10T07:16:44,960 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/recovered.edits/368.seqid, newMaxSeqId=368, maxSeqId=4 2024-12-10T07:16:44,960 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. 2024-12-10T07:16:44,960 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for 6351062f9f9ac7f3c520d90dabe1dd24: Waiting for close lock at 1733815003267Running coprocessor pre-close hooks at 1733815003267Disabling compacts and flushes for region at 1733815003267Disabling writes for close at 1733815003267Obtaining lock to block concurrent updates at 1733815003268 (+1 ms)Preparing flush snapshotting stores in 6351062f9f9ac7f3c520d90dabe1dd24 at 1733815003268Finished memstore snapshotting TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733815003269 (+1 ms)Flushing stores of TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24. at 1733815003270 (+1 ms)Flushing 6351062f9f9ac7f3c520d90dabe1dd24/A: creating writer at 1733815003270Flushing mob file 6351062f9f9ac7f3c520d90dabe1dd24/A: appending metadata at 1733815003278 (+8 ms)Flushing mob file 6351062f9f9ac7f3c520d90dabe1dd24/A: closing flushed file at 1733815003278Flushing 6351062f9f9ac7f3c520d90dabe1dd24/A: appending metadata at 1733815003693 (+415 ms)Flushing 6351062f9f9ac7f3c520d90dabe1dd24/A: closing flushed file at 1733815003693Flushing 6351062f9f9ac7f3c520d90dabe1dd24/B: creating writer at 1733815004109 (+416 ms)Flushing 6351062f9f9ac7f3c520d90dabe1dd24/B: appending metadata at 1733815004113 (+4 ms)Flushing 6351062f9f9ac7f3c520d90dabe1dd24/B: closing flushed file at 1733815004113Flushing 6351062f9f9ac7f3c520d90dabe1dd24/C: creating writer at 1733815004524 (+411 ms)Flushing 6351062f9f9ac7f3c520d90dabe1dd24/C: appending metadata at 1733815004528 (+4 ms)Flushing 6351062f9f9ac7f3c520d90dabe1dd24/C: closing flushed file at 1733815004528Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48a1824a: reopening flushed file at 1733815004942 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fc0a462: reopening flushed file at 1733815004947 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6483b9e7: reopening flushed file at 1733815004951 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 6351062f9f9ac7f3c520d90dabe1dd24 in 1688ms, sequenceid=365, compaction requested=true at 1733815004955 (+4 ms)Writing region close event to WAL at 1733815004956 (+1 ms)Running coprocessor post-close hooks at 1733815004960 (+4 ms)Closed at 1733815004960 2024-12-10T07:16:44,962 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed 6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:44,962 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=6351062f9f9ac7f3c520d90dabe1dd24, regionState=CLOSED 2024-12-10T07:16:44,963 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 because future has completed 2024-12-10T07:16:44,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-10T07:16:44,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure 6351062f9f9ac7f3c520d90dabe1dd24, server=b1396b8d2d12,35871,1733814875835 in 1.8490 sec 2024-12-10T07:16:44,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-12-10T07:16:44,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6351062f9f9ac7f3c520d90dabe1dd24, UNASSIGN in 1.8530 sec 2024-12-10T07:16:44,968 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-10T07:16:44,968 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8560 sec 2024-12-10T07:16:44,969 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733815004968"}]},"ts":"1733815004968"} 2024-12-10T07:16:44,970 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T07:16:44,970 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T07:16:44,971 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8620 sec 2024-12-10T07:16:45,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-10T07:16:45,255 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:16:45,256 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-10T07:16:45,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:16:45,259 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:16:45,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-10T07:16:45,260 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:16:45,264 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:45,267 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A, FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B, FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C, FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/recovered.edits] 2024-12-10T07:16:45,273 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/12633e7d44324e40935ca9f273ea9660 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/12633e7d44324e40935ca9f273ea9660 2024-12-10T07:16:45,273 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/b3fee7d05d5d48cfbab022dcd3a53144 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/b3fee7d05d5d48cfbab022dcd3a53144 2024-12-10T07:16:45,273 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/75ec5d8e77fb403d82e4a69c3375fd7c to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/A/75ec5d8e77fb403d82e4a69c3375fd7c 2024-12-10T07:16:45,277 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/0ef8ad8682724903b2673483b2617529 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/0ef8ad8682724903b2673483b2617529 2024-12-10T07:16:45,277 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/56b198c0eb2441a2893385684e3eefaa to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/56b198c0eb2441a2893385684e3eefaa 2024-12-10T07:16:45,277 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/4b1a257e8a1c413783a7ba4c41573731 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/B/4b1a257e8a1c413783a7ba4c41573731 2024-12-10T07:16:45,281 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/63ebcf3b28f04fe2994dd532713e41df to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/63ebcf3b28f04fe2994dd532713e41df 2024-12-10T07:16:45,281 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/262c0c2bc180405199ccb476b5350117 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/262c0c2bc180405199ccb476b5350117 2024-12-10T07:16:45,281 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/bbe04c6c1e01406bbccfac85e1adac42 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/C/bbe04c6c1e01406bbccfac85e1adac42 2024-12-10T07:16:45,283 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/recovered.edits/368.seqid to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24/recovered.edits/368.seqid 2024-12-10T07:16:45,284 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:45,284 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T07:16:45,284 DEBUG [PEWorker-4 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T07:16:45,285 DEBUG [PEWorker-4 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-10T07:16:45,287 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-10T07:16:45,294 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100cd9709016e5440d9850459d23efafcb_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412100cd9709016e5440d9850459d23efafcb_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:45,294 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412101c9df304aa104ca8a13a49110b2b14dc_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412101c9df304aa104ca8a13a49110b2b14dc_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:45,294 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210251dc20942d441c4bcfb96279c7ba640_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210251dc20942d441c4bcfb96279c7ba640_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:45,294 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412102db89eeb1a3f45ccb3d24348730afbd2_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412102db89eeb1a3f45ccb3d24348730afbd2_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:45,294 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412104906c5c27c444c47879c4208f4aaaf81_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412104906c5c27c444c47879c4208f4aaaf81_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:45,295 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412104ac2d9e8f6ea4bf0910827cdaa2a4b01_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412104ac2d9e8f6ea4bf0910827cdaa2a4b01_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:45,295 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412106cc5b724521e4fb5a57bb90396034f57_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412106cc5b724521e4fb5a57bb90396034f57_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:45,296 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105cd79a8d32b249e5b4c2055128ab02c5_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412105cd79a8d32b249e5b4c2055128ab02c5_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:45,296 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121071004febae4743ec9b960d7cfea9edfc_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121071004febae4743ec9b960d7cfea9edfc_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:45,296 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121072466dff407d41c4857917b286b242c3_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121072466dff407d41c4857917b286b242c3_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:45,297 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121091e04e6a031649ffa03dff999cae5736_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121091e04e6a031649ffa03dff999cae5736_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:45,297 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121084ae884364e5451e9b88791dd3f132c8_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121084ae884364e5451e9b88791dd3f132c8_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:45,297 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121074371cf7693546fcbff887bd39ff396f_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121074371cf7693546fcbff887bd39ff396f_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:45,297 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a57585cfcf074c3ea174484207950769_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a57585cfcf074c3ea174484207950769_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:45,297 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ab3b89cec8f34384a4251c9cd2be34c9_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ab3b89cec8f34384a4251c9cd2be34c9_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:45,298 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210dad699dbf2804e07b74b66661a8fe6d9_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210dad699dbf2804e07b74b66661a8fe6d9_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:45,298 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f6ee4fe1a5b94060be9bb0cde9869a80_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f6ee4fe1a5b94060be9bb0cde9869a80_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:45,298 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f753a0fb9c7249558de262b6e20b8ae2_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210f753a0fb9c7249558de262b6e20b8ae2_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:45,298 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210fc40b51efa3e4f34a60d3d51f47508df_6351062f9f9ac7f3c520d90dabe1dd24 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210fc40b51efa3e4f34a60d3d51f47508df_6351062f9f9ac7f3c520d90dabe1dd24 2024-12-10T07:16:45,298 DEBUG [PEWorker-4 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T07:16:45,300 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:16:45,303 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T07:16:45,305 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T07:16:45,306 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:16:45,306 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T07:16:45,306 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733815005306"}]},"ts":"9223372036854775807"} 2024-12-10T07:16:45,309 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T07:16:45,309 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6351062f9f9ac7f3c520d90dabe1dd24, NAME => 'TestAcidGuarantees,,1733814970274.6351062f9f9ac7f3c520d90dabe1dd24.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T07:16:45,309 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T07:16:45,309 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733815005309"}]},"ts":"9223372036854775807"} 2024-12-10T07:16:45,311 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T07:16:45,312 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:16:45,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 56 msec 2024-12-10T07:16:45,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-10T07:16:45,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T07:16:45,365 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:16:45,376 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=255 (was 258), OpenFileDescriptor=447 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=954 (was 960), ProcessCount=11 (was 11), AvailableMemoryMB=1632 (was 3473) 2024-12-10T07:16:45,386 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=255, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=954, ProcessCount=11, AvailableMemoryMB=1631 2024-12-10T07:16:45,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T07:16:45,388 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T07:16:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T07:16:45,390 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T07:16:45,390 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:45,390 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-12-10T07:16:45,391 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T07:16:45,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-10T07:16:45,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742296_1472 (size=1061) 2024-12-10T07:16:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-10T07:16:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-10T07:16:45,801 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3900c655fa2c79e81a5ed2f352e2cded, NAME => 'TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8 2024-12-10T07:16:45,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742297_1473 (size=53) 2024-12-10T07:16:46,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-10T07:16:46,212 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T07:16:46,212 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3900c655fa2c79e81a5ed2f352e2cded, disabling compactions & flushes 2024-12-10T07:16:46,212 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. 2024-12-10T07:16:46,212 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. 2024-12-10T07:16:46,212 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. after waiting 0 ms 2024-12-10T07:16:46,212 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. 2024-12-10T07:16:46,212 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. 2024-12-10T07:16:46,212 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3900c655fa2c79e81a5ed2f352e2cded: Waiting for close lock at 1733815006212Disabling compacts and flushes for region at 1733815006212Disabling writes for close at 1733815006212Writing region close event to WAL at 1733815006212Closed at 1733815006212 2024-12-10T07:16:46,214 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T07:16:46,215 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733815006214"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733815006214"}]},"ts":"1733815006214"} 2024-12-10T07:16:46,217 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T07:16:46,219 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T07:16:46,219 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733815006219"}]},"ts":"1733815006219"} 2024-12-10T07:16:46,221 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T07:16:46,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3900c655fa2c79e81a5ed2f352e2cded, ASSIGN}] 2024-12-10T07:16:46,222 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3900c655fa2c79e81a5ed2f352e2cded, ASSIGN 2024-12-10T07:16:46,223 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3900c655fa2c79e81a5ed2f352e2cded, ASSIGN; state=OFFLINE, location=b1396b8d2d12,35871,1733814875835; forceNewPlan=false, retain=false 2024-12-10T07:16:46,375 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=3900c655fa2c79e81a5ed2f352e2cded, regionState=OPENING, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:46,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3900c655fa2c79e81a5ed2f352e2cded, ASSIGN because future has completed 2024-12-10T07:16:46,379 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835}] 2024-12-10T07:16:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-10T07:16:46,536 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. 2024-12-10T07:16:46,536 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => 3900c655fa2c79e81a5ed2f352e2cded, NAME => 'TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded.', STARTKEY => '', ENDKEY => ''} 2024-12-10T07:16:46,537 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3900c655fa2c79e81a5ed2f352e2cded 2024-12-10T07:16:46,537 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T07:16:46,537 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for 3900c655fa2c79e81a5ed2f352e2cded 2024-12-10T07:16:46,537 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for 3900c655fa2c79e81a5ed2f352e2cded 2024-12-10T07:16:46,539 INFO [StoreOpener-3900c655fa2c79e81a5ed2f352e2cded-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3900c655fa2c79e81a5ed2f352e2cded 2024-12-10T07:16:46,540 INFO [StoreOpener-3900c655fa2c79e81a5ed2f352e2cded-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:16:46,541 INFO [StoreOpener-3900c655fa2c79e81a5ed2f352e2cded-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3900c655fa2c79e81a5ed2f352e2cded columnFamilyName A 2024-12-10T07:16:46,541 DEBUG [StoreOpener-3900c655fa2c79e81a5ed2f352e2cded-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:46,541 INFO [StoreOpener-3900c655fa2c79e81a5ed2f352e2cded-1 {}] regionserver.HStore(327): Store=3900c655fa2c79e81a5ed2f352e2cded/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:16:46,541 INFO [StoreOpener-3900c655fa2c79e81a5ed2f352e2cded-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3900c655fa2c79e81a5ed2f352e2cded 2024-12-10T07:16:46,543 INFO [StoreOpener-3900c655fa2c79e81a5ed2f352e2cded-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:16:46,543 INFO [StoreOpener-3900c655fa2c79e81a5ed2f352e2cded-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3900c655fa2c79e81a5ed2f352e2cded columnFamilyName B 2024-12-10T07:16:46,543 DEBUG [StoreOpener-3900c655fa2c79e81a5ed2f352e2cded-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:46,544 INFO [StoreOpener-3900c655fa2c79e81a5ed2f352e2cded-1 {}] regionserver.HStore(327): Store=3900c655fa2c79e81a5ed2f352e2cded/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:16:46,544 INFO [StoreOpener-3900c655fa2c79e81a5ed2f352e2cded-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3900c655fa2c79e81a5ed2f352e2cded 2024-12-10T07:16:46,545 INFO [StoreOpener-3900c655fa2c79e81a5ed2f352e2cded-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:16:46,545 INFO [StoreOpener-3900c655fa2c79e81a5ed2f352e2cded-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3900c655fa2c79e81a5ed2f352e2cded columnFamilyName C 2024-12-10T07:16:46,545 DEBUG [StoreOpener-3900c655fa2c79e81a5ed2f352e2cded-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:16:46,546 INFO [StoreOpener-3900c655fa2c79e81a5ed2f352e2cded-1 {}] regionserver.HStore(327): Store=3900c655fa2c79e81a5ed2f352e2cded/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:16:46,546 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. 2024-12-10T07:16:46,546 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for 3900c655fa2c79e81a5ed2f352e2cded 2024-12-10T07:16:46,547 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded 2024-12-10T07:16:46,548 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded 2024-12-10T07:16:46,550 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for 3900c655fa2c79e81a5ed2f352e2cded 2024-12-10T07:16:46,550 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for 3900c655fa2c79e81a5ed2f352e2cded 2024-12-10T07:16:46,550 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T07:16:46,552 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for 3900c655fa2c79e81a5ed2f352e2cded 2024-12-10T07:16:46,555 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T07:16:46,556 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened 3900c655fa2c79e81a5ed2f352e2cded; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73850451, jitterRate=0.10045747458934784}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T07:16:46,556 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3900c655fa2c79e81a5ed2f352e2cded 2024-12-10T07:16:46,557 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for 3900c655fa2c79e81a5ed2f352e2cded: Running coprocessor pre-open hook at 1733815006537Writing region info on filesystem at 1733815006537Initializing all the Stores at 1733815006538 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733815006538Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733815006538Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733815006538Cleaning up temporary data from old regions at 1733815006550 (+12 ms)Running coprocessor post-open hooks at 1733815006556 (+6 ms)Region opened successfully at 1733815006557 (+1 ms) 2024-12-10T07:16:46,558 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., pid=114, masterSystemTime=1733815006531 2024-12-10T07:16:46,559 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. 2024-12-10T07:16:46,559 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. 2024-12-10T07:16:46,560 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=3900c655fa2c79e81a5ed2f352e2cded, regionState=OPEN, openSeqNum=2, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:46,561 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 because future has completed 2024-12-10T07:16:46,563 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-10T07:16:46,563 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure 3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 in 183 msec 2024-12-10T07:16:46,565 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-12-10T07:16:46,565 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3900c655fa2c79e81a5ed2f352e2cded, ASSIGN in 342 msec 2024-12-10T07:16:46,565 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T07:16:46,566 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733815006565"}]},"ts":"1733815006565"} 2024-12-10T07:16:46,567 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T07:16:46,567 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T07:16:46,568 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1790 sec 2024-12-10T07:16:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-10T07:16:47,536 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:16:47,539 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@460158be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,539 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:16:47,539 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:16:47,541 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:16:47,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:16:47,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:16:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38a71148, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:16:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:16:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:47,544 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:16:47,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62bf0aaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,545 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:16:47,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:16:47,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:16:47,547 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:16:47,548 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T07:16:47,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is b1396b8d2d12,36259,1733814874775 2024-12-10T07:16:47,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7508118c 2024-12-10T07:16:47,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T07:16:47,550 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T07:16:47,551 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5159e3f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,551 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:16:47,552 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:16:47,552 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:16:47,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:16:47,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:16:47,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2593ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:16:47,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:16:47,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:47,553 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:16:47,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18353881, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,555 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1062bfdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,555 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:16:47,555 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:16:47,555 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:16:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:16:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:16:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63e65c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:16:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:16:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:47,557 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:16:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a517d36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d7232e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,558 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:16:47,558 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:16:47,558 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:16:47,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:16:47,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:16:47,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30bc0c9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:16:47,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:16:47,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:47,559 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:16:47,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d700dc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,560 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39880b68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,560 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:16:47,560 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:16:47,561 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:16:47,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:16:47,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:16:47,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd71398, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:16:47,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:16:47,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:47,562 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:16:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2445c79d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67f0c5cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,562 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:16:47,562 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:16:47,563 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:16:47,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:16:47,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:16:47,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e74feb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:16:47,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:16:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:47,564 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:16:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d304a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,565 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1593fd05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,565 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:16:47,565 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:16:47,566 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:16:47,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:16:47,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:16:47,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@462a76ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:16:47,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:16:47,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:47,567 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:16:47,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cbfe751, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@729114ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,568 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:16:47,568 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:16:47,568 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:16:47,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:16:47,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:16:47,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@86dbe5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:16:47,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:16:47,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:47,569 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:16:47,570 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45bb3b03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@139f4c00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,570 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:16:47,570 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:16:47,571 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:16:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:16:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:16:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a5d768b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:16:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:16:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:47,572 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:16:47,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149cdc0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14b03bcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,573 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:16:47,573 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:16:47,573 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:16:47,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:16:47,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:16:47,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@438e3ff5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:16:47,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:16:47,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:47,574 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:16:47,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14427083, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b0bbd77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,576 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:16:47,576 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:16:47,576 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:16:47,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:16:47,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:16:47,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62adf910, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:16:47,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:16:47,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:16:47,577 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:16:47,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7460b0d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:16:47,578 DEBUG [Thread-2131 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:16:47,578 DEBUG [Thread-2129 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:16:47,578 DEBUG [Thread-2114 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:16:47,578 DEBUG [Thread-2120 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:16:47,578 DEBUG [Thread-2116 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:16:47,579 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:16:47,579 DEBUG [Thread-2123 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:16:47,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:16:47,579 DEBUG [Thread-2127 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:16:47,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:16:47,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:16:47,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:16:47,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:16:47,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:16:47,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:16:47,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:16:47,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:16:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-10T07:16:47,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:16:47,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:16:47,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:16:47,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:16:47,581 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:16:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T07:16:47,581 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:16:47,581 DEBUG [Thread-2123 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:16:47,581 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:16:47,581 DEBUG [Thread-2112 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:16:47,581 DEBUG [Thread-2123 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:16:47,582 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:16:47,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:16:47,582 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:16:47,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:16:47,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:16:47,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:16:47,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:16:47,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:16:47,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:16:47,583 DEBUG [Thread-2118 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:16:47,583 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:16:47,584 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:16:47,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:16:47,584 DEBUG [Thread-2125 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:16:47,585 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:16:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:16:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:16:47,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:16:47,587 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:16:47,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:16:47,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:16:47,587 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:16:47,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:16:47,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:16:47,589 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:16:47,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:16:47,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2] 2024-12-10T07:16:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 3900c655fa2c79e81a5ed2f352e2cded 2024-12-10T07:16:47,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3900c655fa2c79e81a5ed2f352e2cded 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T07:16:47,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3900c655fa2c79e81a5ed2f352e2cded, store=A 2024-12-10T07:16:47,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:47,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3900c655fa2c79e81a5ed2f352e2cded, store=B 2024-12-10T07:16:47,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:47,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3900c655fa2c79e81a5ed2f352e2cded, store=C 2024-12-10T07:16:47,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:16:47,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/A/bd61468d1a5b4957884d3bff9ea27c4c is 50, key is test_row_1/A:col10/1733815007591/Put/seqid=0 2024-12-10T07:16:47,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742298_1474 (size=7315) 2024-12-10T07:16:47,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733815007604, max timestamp=1733815007641Num uniques 859; 2024-12-10T07:16:47,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-10T07:16:47,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733815007604, max timestamp=1733815007641Num uniques 859; 2024-12-10T07:16:47,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-10T07:16:47,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733815007604, max timestamp=1733815007641Num uniques 859; 2024-12-10T07:16:47,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-10T07:16:47,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52240 deadline: 1733815067662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:47,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:47,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:47,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:47,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52162 deadline: 1733815067664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:47,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52208 deadline: 1733815067664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:47,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52178 deadline: 1733815067664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:47,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52244 deadline: 1733815067670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:47,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:47,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:47,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T07:16:47,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-10T07:16:47,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. 2024-12-10T07:16:47,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. as already flushing 2024-12-10T07:16:47,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. 2024-12-10T07:16:47,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3900c655fa2c79e81a5ed2f352e2cded, NAME => 'TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:47,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3900c655fa2c79e81a5ed2f352e2cded, NAME => 'TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3900c655fa2c79e81a5ed2f352e2cded, NAME => 'TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3900c655fa2c79e81a5ed2f352e2cded, NAME => 'TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:47,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52240 deadline: 1733815067774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:47,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:47,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52162 deadline: 1733815067775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:47,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52178 deadline: 1733815067775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:47,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52208 deadline: 1733815067785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:47,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:47,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:47,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:47,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52244 deadline: 1733815067785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:47,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:47,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:47,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:47,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-10T07:16:47,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. 2024-12-10T07:16:47,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. as already flushing 2024-12-10T07:16:47,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. 2024-12-10T07:16:47,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3900c655fa2c79e81a5ed2f352e2cded, NAME => 'TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:47,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3900c655fa2c79e81a5ed2f352e2cded, NAME => 'TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:47,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3900c655fa2c79e81a5ed2f352e2cded, NAME => 'TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3900c655fa2c79e81a5ed2f352e2cded, NAME => 'TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:47,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T07:16:47,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52240 deadline: 1733815067985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:47,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:47,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:47,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:47,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52162 deadline: 1733815067985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:47,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:47,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:47,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:47,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52178 deadline: 1733815067985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:48,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:48,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52208 deadline: 1733815067994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:48,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:48,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52244 deadline: 1733815067994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:48,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:48,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:48,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/A/bd61468d1a5b4957884d3bff9ea27c4c 2024-12-10T07:16:48,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-10T07:16:48,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. 2024-12-10T07:16:48,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. as already flushing 2024-12-10T07:16:48,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. 2024-12-10T07:16:48,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3900c655fa2c79e81a5ed2f352e2cded, NAME => 'TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:48,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3900c655fa2c79e81a5ed2f352e2cded, NAME => 'TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:48,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3900c655fa2c79e81a5ed2f352e2cded, NAME => 'TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3900c655fa2c79e81a5ed2f352e2cded, NAME => 'TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:16:48,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/B/662fd8d5723e49d6a28197fb2cf599bc is 50, key is test_row_1/B:col10/1733815007591/Put/seqid=0 2024-12-10T07:16:48,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742299_1475 (size=7315) 2024-12-10T07:16:48,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/B/662fd8d5723e49d6a28197fb2cf599bc 2024-12-10T07:16:48,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/C/26111cf35b9747c882d09c641f8f607a is 50, key is test_row_1/C:col10/1733815007591/Put/seqid=0 2024-12-10T07:16:48,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742300_1476 (size=7315) 2024-12-10T07:16:48,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/C/26111cf35b9747c882d09c641f8f607a 2024-12-10T07:16:48,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/A/bd61468d1a5b4957884d3bff9ea27c4c as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/bd61468d1a5b4957884d3bff9ea27c4c 2024-12-10T07:16:48,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/bd61468d1a5b4957884d3bff9ea27c4c, entries=50, sequenceid=11, filesize=7.1 K 2024-12-10T07:16:48,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/B/662fd8d5723e49d6a28197fb2cf599bc as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/662fd8d5723e49d6a28197fb2cf599bc 2024-12-10T07:16:48,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/662fd8d5723e49d6a28197fb2cf599bc, entries=50, sequenceid=11, filesize=7.1 K 2024-12-10T07:16:48,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/C/26111cf35b9747c882d09c641f8f607a as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/26111cf35b9747c882d09c641f8f607a 2024-12-10T07:16:48,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/26111cf35b9747c882d09c641f8f607a, entries=50, sequenceid=11, filesize=7.1 K 2024-12-10T07:16:48,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 3900c655fa2c79e81a5ed2f352e2cded in 515ms, sequenceid=11, compaction requested=false 2024-12-10T07:16:48,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3900c655fa2c79e81a5ed2f352e2cded: 2024-12-10T07:16:48,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-10T07:16:48,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. 2024-12-10T07:16:48,194 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 3900c655fa2c79e81a5ed2f352e2cded 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-12-10T07:16:48,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3900c655fa2c79e81a5ed2f352e2cded, store=A 2024-12-10T07:16:48,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:16:48,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3900c655fa2c79e81a5ed2f352e2cded, store=B 2024-12-10T07:16:48,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:16:48,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3900c655fa2c79e81a5ed2f352e2cded, store=C 2024-12-10T07:16:48,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:16:48,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/A/9d12a8c215594f718319279fcd8924d9 is 50, key is test_row_0/A:col10/1733815007662/Put/seqid=0 2024-12-10T07:16:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T07:16:48,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 3900c655fa2c79e81a5ed2f352e2cded 2024-12-10T07:16:48,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. as already flushing 2024-12-10T07:16:48,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:48,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52244 deadline: 1733815068305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:48,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52240 deadline: 1733815068308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:48,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:48,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52162 deadline: 1733815068309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:48,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52178 deadline: 1733815068309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:48,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:48,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52208 deadline: 1733815068310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:48,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:48,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:48,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52240 deadline: 1733815068415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:48,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:48,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52162 deadline: 1733815068425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:48,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52178 deadline: 1733815068425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:48,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:48,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52208 deadline: 1733815068425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:48,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:48,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:48,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:48,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52240 deadline: 1733815068624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:48,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:48,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:48,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52162 deadline: 1733815068634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:48,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:48,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52178 deadline: 1733815068635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:48,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:48,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:48,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:48,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52208 deadline: 1733815068635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:48,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:48,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T07:16:48,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52244 deadline: 1733815068814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:48,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:48,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:48,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52240 deadline: 1733815068935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:48,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:48,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52162 deadline: 1733815068944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:48,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:48,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52178 deadline: 1733815068944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:48,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:48,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:48,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52208 deadline: 1733815068944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:48,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:48,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:48,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:49,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52240 deadline: 1733815069445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:49,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:49,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:49,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:49,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:49,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52162 deadline: 1733815069455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:49,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:49,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52178 deadline: 1733815069455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:49,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:49,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52208 deadline: 1733815069455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:49,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:49,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:49,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:49,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T07:16:49,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52244 deadline: 1733815069835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:49,841 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_0' in 3900c655fa2c79e81a5ed2f352e2cded of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:49,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:49,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:50,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:50,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52240 deadline: 1733815070465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:50,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:50,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52162 deadline: 1733815070465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:50,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:50,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52208 deadline: 1733815070465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:50,466 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_2' in 3900c655fa2c79e81a5ed2f352e2cded of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:50,466 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_0' in 3900c655fa2c79e81a5ed2f352e2cded of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:50,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:50,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:50,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:50,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:50,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:50,466 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 3900c655fa2c79e81a5ed2f352e2cded of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:50,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:50,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:50,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:50,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:50,478 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T07:16:50,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:50,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52178 deadline: 1733815070474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:50,479 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 3900c655fa2c79e81a5ed2f352e2cded of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:50,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:50,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:50,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T07:16:51,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52244 deadline: 1733815071864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:51,870 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_0' in 3900c655fa2c79e81a5ed2f352e2cded of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:51,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:51,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:51,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:52,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52208 deadline: 1733815072474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:52,479 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 3900c655fa2c79e81a5ed2f352e2cded of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:52,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52240 deadline: 1733815072484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:52,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52162 deadline: 1733815072484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:52,489 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_2' in 3900c655fa2c79e81a5ed2f352e2cded of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:52,489 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_0' in 3900c655fa2c79e81a5ed2f352e2cded of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:52,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:52,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:52,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:52,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:52,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:52,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:52,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52178 deadline: 1733815072494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:52,499 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 3900c655fa2c79e81a5ed2f352e2cded of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:52,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:52,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:52,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:55,287 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-10T07:16:55,287 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-10T07:16:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T07:16:55,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52244 deadline: 1733815075914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:55,920 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_0' in 3900c655fa2c79e81a5ed2f352e2cded of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:55,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:55,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:55,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:56,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52208 deadline: 1733815076494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:56,495 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 3900c655fa2c79e81a5ed2f352e2cded of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:56,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52162 deadline: 1733815076504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:56,510 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_0' in 3900c655fa2c79e81a5ed2f352e2cded of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:56,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:56,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:56,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:56,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:56,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52240 deadline: 1733815076525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:56,528 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_2' in 3900c655fa2c79e81a5ed2f352e2cded of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:56,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:56,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:56,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:56,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:16:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52178 deadline: 1733815076544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:16:56,547 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 3900c655fa2c79e81a5ed2f352e2cded of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:16:56,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:56,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:16:56,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:16:57,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742301_1477 (size=12001) 2024-12-10T07:16:57,719 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/A/9d12a8c215594f718319279fcd8924d9 2024-12-10T07:16:57,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/B/aa49608f244445c0bd29a60d275da0d6 is 50, key is test_row_0/B:col10/1733815007662/Put/seqid=0 2024-12-10T07:16:57,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742302_1478 (size=12001) 2024-12-10T07:16:58,134 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/B/aa49608f244445c0bd29a60d275da0d6 2024-12-10T07:16:58,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/C/b9228f5b59de418188942d192b8c609b is 50, key is test_row_0/C:col10/1733815007662/Put/seqid=0 2024-12-10T07:16:58,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742303_1479 (size=12001) 2024-12-10T07:16:58,805 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/C/b9228f5b59de418188942d192b8c609b 2024-12-10T07:16:58,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/A/9d12a8c215594f718319279fcd8924d9 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/9d12a8c215594f718319279fcd8924d9 2024-12-10T07:16:58,812 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/9d12a8c215594f718319279fcd8924d9, entries=150, sequenceid=45, filesize=11.7 K 2024-12-10T07:16:58,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/B/aa49608f244445c0bd29a60d275da0d6 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/aa49608f244445c0bd29a60d275da0d6 2024-12-10T07:16:58,815 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/aa49608f244445c0bd29a60d275da0d6, entries=150, sequenceid=45, filesize=11.7 K 2024-12-10T07:16:58,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/C/b9228f5b59de418188942d192b8c609b as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/b9228f5b59de418188942d192b8c609b 2024-12-10T07:16:58,819 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/b9228f5b59de418188942d192b8c609b, entries=150, sequenceid=45, filesize=11.7 K 2024-12-10T07:16:58,820 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=46.96 KB/48090 for 3900c655fa2c79e81a5ed2f352e2cded in 10626ms, sequenceid=45, compaction requested=false 2024-12-10T07:16:58,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 3900c655fa2c79e81a5ed2f352e2cded: 2024-12-10T07:16:58,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. 2024-12-10T07:16:58,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-10T07:16:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-10T07:16:58,823 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-10T07:16:58,823 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 11.2390 sec 2024-12-10T07:16:58,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 11.2440 sec 2024-12-10T07:17:03,710 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T07:17:05,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-10T07:17:05,825 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:17:05,825 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:17:05,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-10T07:17:05,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-10T07:17:05,826 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:17:05,827 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:17:05,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:17:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-10T07:17:05,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-10T07:17:05,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. 2024-12-10T07:17:05,979 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 3900c655fa2c79e81a5ed2f352e2cded 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T07:17:05,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3900c655fa2c79e81a5ed2f352e2cded, store=A 2024-12-10T07:17:05,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:05,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3900c655fa2c79e81a5ed2f352e2cded, store=B 2024-12-10T07:17:05,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:05,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3900c655fa2c79e81a5ed2f352e2cded, store=C 2024-12-10T07:17:05,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:05,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/A/c3abfaa39b8c424cb23e93c5d8e23dbc is 50, key is test_row_0/A:col10/1733815008298/Put/seqid=0 2024-12-10T07:17:05,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742304_1480 (size=12001) 2024-12-10T07:17:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 3900c655fa2c79e81a5ed2f352e2cded 2024-12-10T07:17:06,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. as already flushing 2024-12-10T07:17:06,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733815026051, max timestamp=1733815026092Num uniques 859; 2024-12-10T07:17:06,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733815026051, max timestamp=1733815026092Num uniques 859; 2024-12-10T07:17:06,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T07:17:06,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T07:17:06,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733815026051, max timestamp=1733815026092Num uniques 859; 2024-12-10T07:17:06,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T07:17:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-10T07:17:06,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52244 deadline: 1733815086227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:06,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:06,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:06,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:17:06,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52244 deadline: 1733815086345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:06,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:06,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:06,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:17:06,388 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/A/c3abfaa39b8c424cb23e93c5d8e23dbc 2024-12-10T07:17:06,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/B/034ee2a728ad4e1782e5e4dec216eafe is 50, key is test_row_0/B:col10/1733815008298/Put/seqid=0 2024-12-10T07:17:06,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742305_1481 (size=12001) 2024-12-10T07:17:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-10T07:17:06,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52244 deadline: 1733815086555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:06,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:06,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:06,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:17:06,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:06,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52162 deadline: 1733815086575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:06,578 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_0' in 3900c655fa2c79e81a5ed2f352e2cded of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18268 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:17:06,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:06,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:06,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:17:06,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52208 deadline: 1733815086594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:06,603 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 3900c655fa2c79e81a5ed2f352e2cded of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18292 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:17:06,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:06,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:06,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:17:06,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52240 deadline: 1733815086614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:06,620 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_2' in 3900c655fa2c79e81a5ed2f352e2cded of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18311 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:17:06,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:06,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:06,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:17:06,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52178 deadline: 1733815086645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:06,648 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 3900c655fa2c79e81a5ed2f352e2cded of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18338 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:17:06,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:06,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:06,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:17:06,798 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/B/034ee2a728ad4e1782e5e4dec216eafe 2024-12-10T07:17:06,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/C/77b78f174e1c49f7a7d9dac43386b937 is 50, key is test_row_0/C:col10/1733815008298/Put/seqid=0 2024-12-10T07:17:06,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742306_1482 (size=12001) 2024-12-10T07:17:06,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:06,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52244 deadline: 1733815086875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:17:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-10T07:17:07,208 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/C/77b78f174e1c49f7a7d9dac43386b937 2024-12-10T07:17:07,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/A/c3abfaa39b8c424cb23e93c5d8e23dbc as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/c3abfaa39b8c424cb23e93c5d8e23dbc 2024-12-10T07:17:07,227 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/c3abfaa39b8c424cb23e93c5d8e23dbc, entries=150, sequenceid=55, filesize=11.7 K 2024-12-10T07:17:07,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/B/034ee2a728ad4e1782e5e4dec216eafe as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/034ee2a728ad4e1782e5e4dec216eafe 2024-12-10T07:17:07,233 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/034ee2a728ad4e1782e5e4dec216eafe, entries=150, sequenceid=55, filesize=11.7 K 2024-12-10T07:17:07,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/C/77b78f174e1c49f7a7d9dac43386b937 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/77b78f174e1c49f7a7d9dac43386b937 2024-12-10T07:17:07,238 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/77b78f174e1c49f7a7d9dac43386b937, entries=150, sequenceid=55, filesize=11.7 K 2024-12-10T07:17:07,239 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3900c655fa2c79e81a5ed2f352e2cded in 1259ms, sequenceid=55, compaction requested=true 2024-12-10T07:17:07,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 3900c655fa2c79e81a5ed2f352e2cded: 2024-12-10T07:17:07,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. 2024-12-10T07:17:07,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-10T07:17:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-10T07:17:07,246 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-10T07:17:07,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4150 sec 2024-12-10T07:17:07,249 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.4220 sec 2024-12-10T07:17:07,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 3900c655fa2c79e81a5ed2f352e2cded 2024-12-10T07:17:07,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3900c655fa2c79e81a5ed2f352e2cded 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-10T07:17:07,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3900c655fa2c79e81a5ed2f352e2cded, store=A 2024-12-10T07:17:07,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T07:17:07,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3900c655fa2c79e81a5ed2f352e2cded, store=B 2024-12-10T07:17:07,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T07:17:07,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3900c655fa2c79e81a5ed2f352e2cded, store=C 2024-12-10T07:17:07,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-10T07:17:07,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/A/eb90fbc1bc5a4272a349724bd024c8c8 is 50, key is test_row_0/A:col10/1733815027401/Put/seqid=0 2024-12-10T07:17:07,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742307_1483 (size=14341) 2024-12-10T07:17:07,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52244 deadline: 1733815087437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:07,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:07,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:07,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:17:07,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:07,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52244 deadline: 1733815087555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:07,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:07,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:07,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:17:07,592 INFO [Thread-2123 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2123. 2024-12-10T07:17:07,592 DEBUG [Thread-2123 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:17:07,592 DEBUG [Thread-2123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:07,592 DEBUG [Thread-2123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:07,592 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:17:07,592 INFO [Thread-2129 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2129. 2024-12-10T07:17:07,592 DEBUG [Thread-2129 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:17:07,592 DEBUG [Thread-2129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:07,593 DEBUG [Thread-2129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:07,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:17:07,597 INFO [Thread-2125 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2125. 2024-12-10T07:17:07,597 DEBUG [Thread-2125 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:17:07,597 DEBUG [Thread-2125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:07,597 DEBUG [Thread-2125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:07,597 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:17:07,600 INFO [Thread-2127 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2127. 2024-12-10T07:17:07,600 DEBUG [Thread-2127 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:17:07,600 DEBUG [Thread-2127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:07,601 DEBUG [Thread-2127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:07,601 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:17:07,601 INFO [Thread-2131 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2131. 2024-12-10T07:17:07,601 DEBUG [Thread-2131 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:17:07,601 DEBUG [Thread-2131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:07,601 DEBUG [Thread-2131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:07,601 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:17:07,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:07,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52244 deadline: 1733815087775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:07,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:07,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:07,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:17:07,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/A/eb90fbc1bc5a4272a349724bd024c8c8 2024-12-10T07:17:07,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/B/87dab5cf88f6416e8549373aeea18ff7 is 50, key is test_row_0/B:col10/1733815027401/Put/seqid=0 2024-12-10T07:17:07,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742308_1484 (size=12001) 2024-12-10T07:17:07,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-10T07:17:07,975 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:17:08,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:08,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52244 deadline: 1733815088084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:08,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:08,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:08,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:17:08,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/B/87dab5cf88f6416e8549373aeea18ff7 2024-12-10T07:17:08,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/C/92d4ae61b5e8433d8c1bee0258ec78f4 is 50, key is test_row_0/C:col10/1733815027401/Put/seqid=0 2024-12-10T07:17:08,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742309_1485 (size=12001) 2024-12-10T07:17:08,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:08,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52244 deadline: 1733815088594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:08,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 , the old value is region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:08,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:08,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., hostname=b1396b8d2d12,35871,1733814875835, seqNum=2 because the exception is null or not the one we care about 2024-12-10T07:17:08,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/C/92d4ae61b5e8433d8c1bee0258ec78f4 2024-12-10T07:17:08,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/A/eb90fbc1bc5a4272a349724bd024c8c8 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/eb90fbc1bc5a4272a349724bd024c8c8 2024-12-10T07:17:08,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/eb90fbc1bc5a4272a349724bd024c8c8, entries=200, sequenceid=93, filesize=14.0 K 2024-12-10T07:17:08,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/B/87dab5cf88f6416e8549373aeea18ff7 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/87dab5cf88f6416e8549373aeea18ff7 2024-12-10T07:17:08,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/87dab5cf88f6416e8549373aeea18ff7, entries=150, sequenceid=93, filesize=11.7 K 2024-12-10T07:17:08,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/C/92d4ae61b5e8433d8c1bee0258ec78f4 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/92d4ae61b5e8433d8c1bee0258ec78f4 2024-12-10T07:17:08,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/92d4ae61b5e8433d8c1bee0258ec78f4, entries=150, sequenceid=93, filesize=11.7 K 2024-12-10T07:17:08,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 3900c655fa2c79e81a5ed2f352e2cded in 1251ms, sequenceid=93, compaction requested=true 2024-12-10T07:17:08,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3900c655fa2c79e81a5ed2f352e2cded: 2024-12-10T07:17:08,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3900c655fa2c79e81a5ed2f352e2cded:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:17:08,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:17:08,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3900c655fa2c79e81a5ed2f352e2cded:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:17:08,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:17:08,654 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:17:08,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3900c655fa2c79e81a5ed2f352e2cded:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:17:08,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:17:08,654 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:17:08,655 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 43318 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:17:08,655 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45658 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:17:08,655 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 3900c655fa2c79e81a5ed2f352e2cded/B is initiating minor compaction (all files) 2024-12-10T07:17:08,655 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 3900c655fa2c79e81a5ed2f352e2cded/A is initiating minor compaction (all files) 2024-12-10T07:17:08,655 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3900c655fa2c79e81a5ed2f352e2cded/A in TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. 2024-12-10T07:17:08,655 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3900c655fa2c79e81a5ed2f352e2cded/B in TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. 2024-12-10T07:17:08,655 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/662fd8d5723e49d6a28197fb2cf599bc, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/aa49608f244445c0bd29a60d275da0d6, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/034ee2a728ad4e1782e5e4dec216eafe, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/87dab5cf88f6416e8549373aeea18ff7] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp, totalSize=42.3 K 2024-12-10T07:17:08,655 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/bd61468d1a5b4957884d3bff9ea27c4c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/9d12a8c215594f718319279fcd8924d9, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/c3abfaa39b8c424cb23e93c5d8e23dbc, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/eb90fbc1bc5a4272a349724bd024c8c8] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp, totalSize=44.6 K 2024-12-10T07:17:08,655 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd61468d1a5b4957884d3bff9ea27c4c, keycount=50, bloomtype=ROW, size=7.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733815007591 2024-12-10T07:17:08,655 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 662fd8d5723e49d6a28197fb2cf599bc, keycount=50, bloomtype=ROW, size=7.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733815007591 2024-12-10T07:17:08,655 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting aa49608f244445c0bd29a60d275da0d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733815007662 2024-12-10T07:17:08,655 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d12a8c215594f718319279fcd8924d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733815007662 2024-12-10T07:17:08,656 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 034ee2a728ad4e1782e5e4dec216eafe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733815008298 2024-12-10T07:17:08,656 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3abfaa39b8c424cb23e93c5d8e23dbc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733815008298 2024-12-10T07:17:08,656 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 87dab5cf88f6416e8549373aeea18ff7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733815026212 2024-12-10T07:17:08,656 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb90fbc1bc5a4272a349724bd024c8c8, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733815026201 2024-12-10T07:17:08,661 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3900c655fa2c79e81a5ed2f352e2cded#B#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:17:08,661 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3900c655fa2c79e81a5ed2f352e2cded#A#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:17:08,661 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/B/f5265b5d4c3c4d48b6178d284b648ff3 is 50, key is test_row_0/B:col10/1733815027401/Put/seqid=0 2024-12-10T07:17:08,662 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/A/ef9254e2dad843e5b96b15fb388203ab is 50, key is test_row_0/A:col10/1733815027401/Put/seqid=0 2024-12-10T07:17:08,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742310_1486 (size=12139) 2024-12-10T07:17:08,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742311_1487 (size=12139) 2024-12-10T07:17:09,068 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/A/ef9254e2dad843e5b96b15fb388203ab as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/ef9254e2dad843e5b96b15fb388203ab 2024-12-10T07:17:09,068 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/B/f5265b5d4c3c4d48b6178d284b648ff3 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/f5265b5d4c3c4d48b6178d284b648ff3 2024-12-10T07:17:09,071 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3900c655fa2c79e81a5ed2f352e2cded/B of 3900c655fa2c79e81a5ed2f352e2cded into f5265b5d4c3c4d48b6178d284b648ff3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:17:09,071 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3900c655fa2c79e81a5ed2f352e2cded/A of 3900c655fa2c79e81a5ed2f352e2cded into ef9254e2dad843e5b96b15fb388203ab(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:17:09,071 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3900c655fa2c79e81a5ed2f352e2cded: 2024-12-10T07:17:09,071 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3900c655fa2c79e81a5ed2f352e2cded: 2024-12-10T07:17:09,071 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., storeName=3900c655fa2c79e81a5ed2f352e2cded/A, priority=12, startTime=1733815028654; duration=0sec 2024-12-10T07:17:09,071 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., storeName=3900c655fa2c79e81a5ed2f352e2cded/B, priority=12, startTime=1733815028654; duration=0sec 2024-12-10T07:17:09,071 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:17:09,071 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:17:09,071 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3900c655fa2c79e81a5ed2f352e2cded:B 2024-12-10T07:17:09,071 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3900c655fa2c79e81a5ed2f352e2cded:A 2024-12-10T07:17:09,071 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:17:09,072 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 43318 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:17:09,072 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 3900c655fa2c79e81a5ed2f352e2cded/C is initiating minor compaction (all files) 2024-12-10T07:17:09,072 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3900c655fa2c79e81a5ed2f352e2cded/C in TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. 2024-12-10T07:17:09,072 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/26111cf35b9747c882d09c641f8f607a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/b9228f5b59de418188942d192b8c609b, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/77b78f174e1c49f7a7d9dac43386b937, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/92d4ae61b5e8433d8c1bee0258ec78f4] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp, totalSize=42.3 K 2024-12-10T07:17:09,073 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 26111cf35b9747c882d09c641f8f607a, keycount=50, bloomtype=ROW, size=7.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733815007591 2024-12-10T07:17:09,073 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting b9228f5b59de418188942d192b8c609b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733815007662 2024-12-10T07:17:09,073 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 77b78f174e1c49f7a7d9dac43386b937, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733815008298 2024-12-10T07:17:09,073 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 92d4ae61b5e8433d8c1bee0258ec78f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733815026212 2024-12-10T07:17:09,081 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3900c655fa2c79e81a5ed2f352e2cded#C#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:17:09,081 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/C/4e754b7dc4884e1abdd031b806c7ae92 is 50, key is test_row_0/C:col10/1733815027401/Put/seqid=0 2024-12-10T07:17:09,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742312_1488 (size=12139) 2024-12-10T07:17:09,489 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/C/4e754b7dc4884e1abdd031b806c7ae92 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/4e754b7dc4884e1abdd031b806c7ae92 2024-12-10T07:17:09,493 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3900c655fa2c79e81a5ed2f352e2cded/C of 3900c655fa2c79e81a5ed2f352e2cded into 4e754b7dc4884e1abdd031b806c7ae92(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:17:09,493 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3900c655fa2c79e81a5ed2f352e2cded: 2024-12-10T07:17:09,493 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., storeName=3900c655fa2c79e81a5ed2f352e2cded/C, priority=12, startTime=1733815028654; duration=0sec 2024-12-10T07:17:09,493 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:17:09,493 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3900c655fa2c79e81a5ed2f352e2cded:C 2024-12-10T07:17:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 3900c655fa2c79e81a5ed2f352e2cded 2024-12-10T07:17:09,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3900c655fa2c79e81a5ed2f352e2cded 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T07:17:09,606 INFO [Thread-2118 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2118. 2024-12-10T07:17:09,606 DEBUG [Thread-2118 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:17:09,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3900c655fa2c79e81a5ed2f352e2cded, store=A 2024-12-10T07:17:09,606 DEBUG [Thread-2118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:09,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:09,606 DEBUG [Thread-2118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:09,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3900c655fa2c79e81a5ed2f352e2cded, store=B 2024-12-10T07:17:09,606 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:17:09,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:09,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3900c655fa2c79e81a5ed2f352e2cded, store=C 2024-12-10T07:17:09,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:09,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/A/c92f3375c5704ca7a663c37a3164cd3a is 50, key is test_row_0/A:col10/1733815027411/Put/seqid=0 2024-12-10T07:17:09,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742313_1489 (size=12001) 2024-12-10T07:17:10,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/A/c92f3375c5704ca7a663c37a3164cd3a 2024-12-10T07:17:10,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/B/daca0668e39a43b18d0ba91c2543ceb1 is 50, key is test_row_0/B:col10/1733815027411/Put/seqid=0 2024-12-10T07:17:10,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742314_1490 (size=12001) 2024-12-10T07:17:10,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/B/daca0668e39a43b18d0ba91c2543ceb1 2024-12-10T07:17:10,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/C/1cf5744883c44682a48c6d562ed2c605 is 50, key is test_row_0/C:col10/1733815027411/Put/seqid=0 2024-12-10T07:17:10,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742315_1491 (size=12001) 2024-12-10T07:17:10,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/C/1cf5744883c44682a48c6d562ed2c605 2024-12-10T07:17:10,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/A/c92f3375c5704ca7a663c37a3164cd3a as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/c92f3375c5704ca7a663c37a3164cd3a 2024-12-10T07:17:10,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/c92f3375c5704ca7a663c37a3164cd3a, entries=150, sequenceid=106, filesize=11.7 K 2024-12-10T07:17:10,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/B/daca0668e39a43b18d0ba91c2543ceb1 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/daca0668e39a43b18d0ba91c2543ceb1 2024-12-10T07:17:10,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/daca0668e39a43b18d0ba91c2543ceb1, entries=150, sequenceid=106, filesize=11.7 K 2024-12-10T07:17:10,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/C/1cf5744883c44682a48c6d562ed2c605 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/1cf5744883c44682a48c6d562ed2c605 2024-12-10T07:17:10,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/1cf5744883c44682a48c6d562ed2c605, entries=150, sequenceid=106, filesize=11.7 K 2024-12-10T07:17:10,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 3900c655fa2c79e81a5ed2f352e2cded in 1238ms, sequenceid=106, compaction requested=false 2024-12-10T07:17:10,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3900c655fa2c79e81a5ed2f352e2cded: 2024-12-10T07:17:16,626 INFO [Thread-2116 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2116. 2024-12-10T07:17:16,626 DEBUG [Thread-2116 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:17:16,626 DEBUG [Thread-2116 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:16,627 DEBUG [Thread-2116 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:16,627 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:17:16,656 INFO [Thread-2112 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2112. 2024-12-10T07:17:16,656 DEBUG [Thread-2112 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:17:16,656 DEBUG [Thread-2112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:16,656 DEBUG [Thread-2112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:16,656 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:17:16,716 INFO [Thread-2114 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2114. 2024-12-10T07:17:16,717 DEBUG [Thread-2114 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:17:16,717 DEBUG [Thread-2114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:16,717 DEBUG [Thread-2114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:16,717 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:17:16,736 INFO [Thread-2120 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2120. 2024-12-10T07:17:16,736 DEBUG [Thread-2120 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:17:16,736 DEBUG [Thread-2120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:16,736 DEBUG [Thread-2120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:16,736 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:17:16,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T07:17:16,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-12-10T07:17:16,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-12-10T07:17:16,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-12-10T07:17:16,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-12-10T07:17:16,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-12-10T07:17:16,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T07:17:16,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T07:17:16,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2001 2024-12-10T07:17:16,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5996 rows 2024-12-10T07:17:16,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1984 2024-12-10T07:17:16,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5950 rows 2024-12-10T07:17:16,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2003 2024-12-10T07:17:16,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6005 rows 2024-12-10T07:17:16,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1990 2024-12-10T07:17:16,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5966 rows 2024-12-10T07:17:16,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2004 2024-12-10T07:17:16,736 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6004 rows 2024-12-10T07:17:16,736 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T07:17:16,736 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T07:17:16,736 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:16,737 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:16,737 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:17:16,737 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-10T07:17:16,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T07:17:16,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-10T07:17:16,740 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733815036740"}]},"ts":"1733815036740"} 2024-12-10T07:17:16,741 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T07:17:16,741 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T07:17:16,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T07:17:16,743 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3900c655fa2c79e81a5ed2f352e2cded, UNASSIGN}] 2024-12-10T07:17:16,744 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3900c655fa2c79e81a5ed2f352e2cded, UNASSIGN 2024-12-10T07:17:16,744 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=3900c655fa2c79e81a5ed2f352e2cded, regionState=CLOSING, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:16,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3900c655fa2c79e81a5ed2f352e2cded, UNASSIGN because future has completed 2024-12-10T07:17:16,746 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T07:17:16,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835}] 2024-12-10T07:17:16,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-10T07:17:16,898 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(122): Close 3900c655fa2c79e81a5ed2f352e2cded 2024-12-10T07:17:16,898 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T07:17:16,898 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1722): Closing 3900c655fa2c79e81a5ed2f352e2cded, disabling compactions & flushes 2024-12-10T07:17:16,898 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. 2024-12-10T07:17:16,898 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. 2024-12-10T07:17:16,898 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. after waiting 0 ms 2024-12-10T07:17:16,898 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. 2024-12-10T07:17:16,898 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(2902): Flushing 3900c655fa2c79e81a5ed2f352e2cded 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-10T07:17:16,899 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3900c655fa2c79e81a5ed2f352e2cded, store=A 2024-12-10T07:17:16,899 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:16,899 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3900c655fa2c79e81a5ed2f352e2cded, store=B 2024-12-10T07:17:16,899 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:16,899 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3900c655fa2c79e81a5ed2f352e2cded, store=C 2024-12-10T07:17:16,899 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:16,929 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/A/336beee06d6f4eb3830ead4d927ee22c is 50, key is test_row_0/A:col10/1733815036625/Put/seqid=0 2024-12-10T07:17:16,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742316_1492 (size=12001) 2024-12-10T07:17:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-10T07:17:17,333 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/A/336beee06d6f4eb3830ead4d927ee22c 2024-12-10T07:17:17,339 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/B/6ec0767c19e043a193cc891ec4b5d840 is 50, key is test_row_0/B:col10/1733815036625/Put/seqid=0 2024-12-10T07:17:17,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742317_1493 (size=12001) 2024-12-10T07:17:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-10T07:17:17,743 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/B/6ec0767c19e043a193cc891ec4b5d840 2024-12-10T07:17:17,749 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/C/75d95b30efbc44a3b42dc75256db2968 is 50, key is test_row_0/C:col10/1733815036625/Put/seqid=0 2024-12-10T07:17:17,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742318_1494 (size=12001) 2024-12-10T07:17:17,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-10T07:17:18,191 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/C/75d95b30efbc44a3b42dc75256db2968 2024-12-10T07:17:18,195 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/A/336beee06d6f4eb3830ead4d927ee22c as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/336beee06d6f4eb3830ead4d927ee22c 2024-12-10T07:17:18,200 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/336beee06d6f4eb3830ead4d927ee22c, entries=150, sequenceid=113, filesize=11.7 K 2024-12-10T07:17:18,201 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/B/6ec0767c19e043a193cc891ec4b5d840 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/6ec0767c19e043a193cc891ec4b5d840 2024-12-10T07:17:18,204 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/6ec0767c19e043a193cc891ec4b5d840, entries=150, sequenceid=113, filesize=11.7 K 2024-12-10T07:17:18,204 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/.tmp/C/75d95b30efbc44a3b42dc75256db2968 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/75d95b30efbc44a3b42dc75256db2968 2024-12-10T07:17:18,207 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/75d95b30efbc44a3b42dc75256db2968, entries=150, sequenceid=113, filesize=11.7 K 2024-12-10T07:17:18,208 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 3900c655fa2c79e81a5ed2f352e2cded in 1310ms, sequenceid=113, compaction requested=true 2024-12-10T07:17:18,208 DEBUG [StoreCloser-TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/bd61468d1a5b4957884d3bff9ea27c4c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/9d12a8c215594f718319279fcd8924d9, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/c3abfaa39b8c424cb23e93c5d8e23dbc, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/eb90fbc1bc5a4272a349724bd024c8c8] to archive 2024-12-10T07:17:18,209 DEBUG [StoreCloser-TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T07:17:18,211 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/9d12a8c215594f718319279fcd8924d9 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/9d12a8c215594f718319279fcd8924d9 2024-12-10T07:17:18,211 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/bd61468d1a5b4957884d3bff9ea27c4c to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/bd61468d1a5b4957884d3bff9ea27c4c 2024-12-10T07:17:18,212 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/c3abfaa39b8c424cb23e93c5d8e23dbc to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/c3abfaa39b8c424cb23e93c5d8e23dbc 2024-12-10T07:17:18,212 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/eb90fbc1bc5a4272a349724bd024c8c8 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/eb90fbc1bc5a4272a349724bd024c8c8 2024-12-10T07:17:18,213 DEBUG [StoreCloser-TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/662fd8d5723e49d6a28197fb2cf599bc, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/aa49608f244445c0bd29a60d275da0d6, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/034ee2a728ad4e1782e5e4dec216eafe, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/87dab5cf88f6416e8549373aeea18ff7] to archive 2024-12-10T07:17:18,213 DEBUG [StoreCloser-TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T07:17:18,215 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/662fd8d5723e49d6a28197fb2cf599bc to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/662fd8d5723e49d6a28197fb2cf599bc 2024-12-10T07:17:18,215 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/aa49608f244445c0bd29a60d275da0d6 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/aa49608f244445c0bd29a60d275da0d6 2024-12-10T07:17:18,215 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/034ee2a728ad4e1782e5e4dec216eafe to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/034ee2a728ad4e1782e5e4dec216eafe 2024-12-10T07:17:18,215 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/87dab5cf88f6416e8549373aeea18ff7 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/87dab5cf88f6416e8549373aeea18ff7 2024-12-10T07:17:18,216 DEBUG [StoreCloser-TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/26111cf35b9747c882d09c641f8f607a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/b9228f5b59de418188942d192b8c609b, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/77b78f174e1c49f7a7d9dac43386b937, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/92d4ae61b5e8433d8c1bee0258ec78f4] to archive 2024-12-10T07:17:18,217 DEBUG [StoreCloser-TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T07:17:18,218 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/92d4ae61b5e8433d8c1bee0258ec78f4 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/92d4ae61b5e8433d8c1bee0258ec78f4 2024-12-10T07:17:18,218 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/77b78f174e1c49f7a7d9dac43386b937 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/77b78f174e1c49f7a7d9dac43386b937 2024-12-10T07:17:18,218 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/b9228f5b59de418188942d192b8c609b to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/b9228f5b59de418188942d192b8c609b 2024-12-10T07:17:18,218 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/26111cf35b9747c882d09c641f8f607a to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/26111cf35b9747c882d09c641f8f607a 2024-12-10T07:17:18,222 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/recovered.edits/116.seqid, newMaxSeqId=116, maxSeqId=1 2024-12-10T07:17:18,223 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. 2024-12-10T07:17:18,223 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1676): Region close journal for 3900c655fa2c79e81a5ed2f352e2cded: Waiting for close lock at 1733815036898Running coprocessor pre-close hooks at 1733815036898Disabling compacts and flushes for region at 1733815036898Disabling writes for close at 1733815036898Obtaining lock to block concurrent updates at 1733815036898Preparing flush snapshotting stores in 3900c655fa2c79e81a5ed2f352e2cded at 1733815036898Finished memstore snapshotting TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733815036899 (+1 ms)Flushing stores of TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded. at 1733815036926 (+27 ms)Flushing 3900c655fa2c79e81a5ed2f352e2cded/A: creating writer at 1733815036926Flushing 3900c655fa2c79e81a5ed2f352e2cded/A: appending metadata at 1733815036929 (+3 ms)Flushing 3900c655fa2c79e81a5ed2f352e2cded/A: closing flushed file at 1733815036929Flushing 3900c655fa2c79e81a5ed2f352e2cded/B: creating writer at 1733815037337 (+408 ms)Flushing 3900c655fa2c79e81a5ed2f352e2cded/B: appending metadata at 1733815037339 (+2 ms)Flushing 3900c655fa2c79e81a5ed2f352e2cded/B: closing flushed file at 1733815037339Flushing 3900c655fa2c79e81a5ed2f352e2cded/C: creating writer at 1733815037746 (+407 ms)Flushing 3900c655fa2c79e81a5ed2f352e2cded/C: appending metadata at 1733815037749 (+3 ms)Flushing 3900c655fa2c79e81a5ed2f352e2cded/C: closing flushed file at 1733815037749Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2121d614: reopening flushed file at 1733815038194 (+445 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@101bfbaf: reopening flushed file at 1733815038200 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@730fbf5: reopening flushed file at 1733815038204 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 3900c655fa2c79e81a5ed2f352e2cded in 1310ms, sequenceid=113, compaction requested=true at 1733815038208 (+4 ms)Writing region close event to WAL at 1733815038219 (+11 ms)Running coprocessor post-close hooks at 1733815038223 (+4 ms)Closed at 1733815038223 2024-12-10T07:17:18,224 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(157): Closed 3900c655fa2c79e81a5ed2f352e2cded 2024-12-10T07:17:18,225 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=3900c655fa2c79e81a5ed2f352e2cded, regionState=CLOSED 2024-12-10T07:17:18,226 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 because future has completed 2024-12-10T07:17:18,228 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-10T07:17:18,228 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; CloseRegionProcedure 3900c655fa2c79e81a5ed2f352e2cded, server=b1396b8d2d12,35871,1733814875835 in 1.4810 sec 2024-12-10T07:17:18,229 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-12-10T07:17:18,229 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3900c655fa2c79e81a5ed2f352e2cded, UNASSIGN in 1.4850 sec 2024-12-10T07:17:18,231 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-10T07:17:18,231 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4870 sec 2024-12-10T07:17:18,232 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733815038232"}]},"ts":"1733815038232"} 2024-12-10T07:17:18,233 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T07:17:18,233 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T07:17:18,234 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4960 sec 2024-12-10T07:17:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-10T07:17:18,885 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:17:18,886 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-10T07:17:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:17:18,890 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:17:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-10T07:17:18,892 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=123, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:17:18,895 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded 2024-12-10T07:17:18,898 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A, FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B, FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C, FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/recovered.edits] 2024-12-10T07:17:18,902 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/c92f3375c5704ca7a663c37a3164cd3a to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/c92f3375c5704ca7a663c37a3164cd3a 2024-12-10T07:17:18,902 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/336beee06d6f4eb3830ead4d927ee22c to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/336beee06d6f4eb3830ead4d927ee22c 2024-12-10T07:17:18,902 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/ef9254e2dad843e5b96b15fb388203ab to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/A/ef9254e2dad843e5b96b15fb388203ab 2024-12-10T07:17:18,905 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/6ec0767c19e043a193cc891ec4b5d840 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/6ec0767c19e043a193cc891ec4b5d840 2024-12-10T07:17:18,906 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/daca0668e39a43b18d0ba91c2543ceb1 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/daca0668e39a43b18d0ba91c2543ceb1 2024-12-10T07:17:18,906 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/f5265b5d4c3c4d48b6178d284b648ff3 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/B/f5265b5d4c3c4d48b6178d284b648ff3 2024-12-10T07:17:18,910 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/4e754b7dc4884e1abdd031b806c7ae92 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/4e754b7dc4884e1abdd031b806c7ae92 2024-12-10T07:17:18,910 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/1cf5744883c44682a48c6d562ed2c605 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/1cf5744883c44682a48c6d562ed2c605 2024-12-10T07:17:18,910 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/75d95b30efbc44a3b42dc75256db2968 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/C/75d95b30efbc44a3b42dc75256db2968 2024-12-10T07:17:18,913 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/recovered.edits/116.seqid to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded/recovered.edits/116.seqid 2024-12-10T07:17:18,914 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/3900c655fa2c79e81a5ed2f352e2cded 2024-12-10T07:17:18,914 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T07:17:18,915 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=123, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:17:18,918 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T07:17:18,919 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T07:17:18,920 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=123, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:17:18,920 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T07:17:18,920 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733815038920"}]},"ts":"9223372036854775807"} 2024-12-10T07:17:18,922 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T07:17:18,922 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3900c655fa2c79e81a5ed2f352e2cded, NAME => 'TestAcidGuarantees,,1733815005388.3900c655fa2c79e81a5ed2f352e2cded.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T07:17:18,922 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T07:17:18,922 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733815038922"}]},"ts":"9223372036854775807"} 2024-12-10T07:17:18,924 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T07:17:18,925 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=123, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:17:18,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 39 msec 2024-12-10T07:17:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-10T07:17:18,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T07:17:18,995 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:17:19,002 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=254 (was 255), OpenFileDescriptor=443 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=820 (was 954), ProcessCount=11 (was 11), AvailableMemoryMB=3754 (was 1631) - AvailableMemoryMB LEAK? - 2024-12-10T07:17:19,009 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=254, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=820, ProcessCount=11, AvailableMemoryMB=3754 2024-12-10T07:17:19,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T07:17:19,011 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T07:17:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-10T07:17:19,012 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T07:17:19,012 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:17:19,012 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 124 2024-12-10T07:17:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-12-10T07:17:19,013 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T07:17:19,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742319_1495 (size=1061) 2024-12-10T07:17:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-12-10T07:17:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-12-10T07:17:19,421 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8 2024-12-10T07:17:19,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742320_1496 (size=53) 2024-12-10T07:17:19,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-12-10T07:17:19,830 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T07:17:19,831 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 508b45081a20fbcb4fc7da445cb3ab9f, disabling compactions & flushes 2024-12-10T07:17:19,831 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:19,831 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:19,831 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. after waiting 0 ms 2024-12-10T07:17:19,831 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:19,831 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:19,831 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 508b45081a20fbcb4fc7da445cb3ab9f: Waiting for close lock at 1733815039830Disabling compacts and flushes for region at 1733815039830Disabling writes for close at 1733815039831 (+1 ms)Writing region close event to WAL at 1733815039831Closed at 1733815039831 2024-12-10T07:17:19,832 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T07:17:19,832 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733815039832"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733815039832"}]},"ts":"1733815039832"} 2024-12-10T07:17:19,833 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T07:17:19,834 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T07:17:19,834 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733815039834"}]},"ts":"1733815039834"} 2024-12-10T07:17:19,835 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-10T07:17:19,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=508b45081a20fbcb4fc7da445cb3ab9f, ASSIGN}] 2024-12-10T07:17:19,836 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=508b45081a20fbcb4fc7da445cb3ab9f, ASSIGN 2024-12-10T07:17:19,836 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=508b45081a20fbcb4fc7da445cb3ab9f, ASSIGN; state=OFFLINE, location=b1396b8d2d12,35871,1733814875835; forceNewPlan=false, retain=false 2024-12-10T07:17:19,987 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=508b45081a20fbcb4fc7da445cb3ab9f, regionState=OPENING, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:19,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=508b45081a20fbcb4fc7da445cb3ab9f, ASSIGN because future has completed 2024-12-10T07:17:19,989 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure 508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835}] 2024-12-10T07:17:20,144 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:20,144 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7752): Opening region: {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} 2024-12-10T07:17:20,144 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:20,144 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T07:17:20,144 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7794): checking encryption for 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:20,144 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7797): checking classloading for 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:20,146 INFO [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:20,147 INFO [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:17:20,148 INFO [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 508b45081a20fbcb4fc7da445cb3ab9f columnFamilyName A 2024-12-10T07:17:20,148 DEBUG [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:17:20,148 INFO [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] regionserver.HStore(327): Store=508b45081a20fbcb4fc7da445cb3ab9f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:17:20,148 INFO [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:20,149 INFO [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:17:20,150 INFO [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 508b45081a20fbcb4fc7da445cb3ab9f columnFamilyName B 2024-12-10T07:17:20,150 DEBUG [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:17:20,150 INFO [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] regionserver.HStore(327): Store=508b45081a20fbcb4fc7da445cb3ab9f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:17:20,150 INFO [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:20,151 INFO [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:17:20,151 INFO [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 508b45081a20fbcb4fc7da445cb3ab9f columnFamilyName C 2024-12-10T07:17:20,151 DEBUG [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:17:20,152 INFO [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] regionserver.HStore(327): Store=508b45081a20fbcb4fc7da445cb3ab9f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:17:20,152 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:20,152 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1038): replaying wal for 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:20,153 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:20,153 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:20,154 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1048): stopping wal replay for 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:20,154 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1060): Cleaning up temporary data for 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:20,155 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T07:17:20,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-12-10T07:17:20,156 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1093): writing seq id for 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:20,158 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T07:17:20,158 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1114): Opened 508b45081a20fbcb4fc7da445cb3ab9f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72171774, jitterRate=0.07544323801994324}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T07:17:20,158 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:20,159 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1006): Region open journal for 508b45081a20fbcb4fc7da445cb3ab9f: Running coprocessor pre-open hook at 1733815040144Writing region info on filesystem at 1733815040144Initializing all the Stores at 1733815040146 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733815040146Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733815040146Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733815040146Cleaning up temporary data from old regions at 1733815040154 (+8 ms)Running coprocessor post-open hooks at 1733815040158 (+4 ms)Region opened successfully at 1733815040159 (+1 ms) 2024-12-10T07:17:20,160 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., pid=126, masterSystemTime=1733815040141 2024-12-10T07:17:20,162 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:20,162 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:20,162 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=508b45081a20fbcb4fc7da445cb3ab9f, regionState=OPEN, openSeqNum=2, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:20,164 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure 508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 because future has completed 2024-12-10T07:17:20,169 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-10T07:17:20,169 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; OpenRegionProcedure 508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 in 177 msec 2024-12-10T07:17:20,172 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-12-10T07:17:20,172 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=508b45081a20fbcb4fc7da445cb3ab9f, ASSIGN in 334 msec 2024-12-10T07:17:20,172 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T07:17:20,172 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733815040172"}]},"ts":"1733815040172"} 2024-12-10T07:17:20,174 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-10T07:17:20,175 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T07:17:20,178 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1640 sec 2024-12-10T07:17:21,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-12-10T07:17:21,175 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:17:21,177 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3af38ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,177 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:17:21,177 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:17:21,178 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:17:21,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:17:21,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:17:21,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a3b5256, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:17:21,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:17:21,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:21,180 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:17:21,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56465f24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,181 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:17:21,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:17:21,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:17:21,183 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:17:21,184 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T07:17:21,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is b1396b8d2d12,36259,1733814874775 2024-12-10T07:17:21,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@465a6e3a 2024-12-10T07:17:21,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T07:17:21,186 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T07:17:21,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-10T07:17:21,188 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T07:17:21,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-10T07:17:21,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742321_1497 (size=1097) 2024-12-10T07:17:21,203 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-10T07:17:21,204 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-10T07:17:21,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T07:17:21,208 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=508b45081a20fbcb4fc7da445cb3ab9f, REOPEN/MOVE}] 2024-12-10T07:17:21,209 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=508b45081a20fbcb4fc7da445cb3ab9f, REOPEN/MOVE 2024-12-10T07:17:21,210 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=508b45081a20fbcb4fc7da445cb3ab9f, regionState=CLOSING, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:21,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=508b45081a20fbcb4fc7da445cb3ab9f, REOPEN/MOVE because future has completed 2024-12-10T07:17:21,211 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T07:17:21,211 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure 508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835}] 2024-12-10T07:17:21,364 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(122): Close 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:21,364 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T07:17:21,364 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1722): Closing 508b45081a20fbcb4fc7da445cb3ab9f, disabling compactions & flushes 2024-12-10T07:17:21,365 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:21,365 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:21,365 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. after waiting 0 ms 2024-12-10T07:17:21,365 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:21,369 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-10T07:17:21,370 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:21,370 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1676): Region close journal for 508b45081a20fbcb4fc7da445cb3ab9f: Waiting for close lock at 1733815041364Running coprocessor pre-close hooks at 1733815041364Disabling compacts and flushes for region at 1733815041364Disabling writes for close at 1733815041365 (+1 ms)Writing region close event to WAL at 1733815041365Running coprocessor post-close hooks at 1733815041370 (+5 ms)Closed at 1733815041370 2024-12-10T07:17:21,370 WARN [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionServer(3299): Not adding moved region record: 508b45081a20fbcb4fc7da445cb3ab9f to self. 2024-12-10T07:17:21,372 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(157): Closed 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:21,373 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=508b45081a20fbcb4fc7da445cb3ab9f, regionState=CLOSED 2024-12-10T07:17:21,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure 508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 because future has completed 2024-12-10T07:17:21,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-10T07:17:21,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseRegionProcedure 508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 in 164 msec 2024-12-10T07:17:21,377 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=508b45081a20fbcb4fc7da445cb3ab9f, REOPEN/MOVE; state=CLOSED, location=b1396b8d2d12,35871,1733814875835; forceNewPlan=false, retain=true 2024-12-10T07:17:21,528 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=508b45081a20fbcb4fc7da445cb3ab9f, regionState=OPENING, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:21,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=508b45081a20fbcb4fc7da445cb3ab9f, REOPEN/MOVE because future has completed 2024-12-10T07:17:21,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=131, ppid=129, state=RUNNABLE, hasLock=false; OpenRegionProcedure 508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835}] 2024-12-10T07:17:21,685 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:21,685 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(7752): Opening region: {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} 2024-12-10T07:17:21,686 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:21,686 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T07:17:21,686 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(7794): checking encryption for 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:21,686 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(7797): checking classloading for 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:21,692 INFO [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:21,693 INFO [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:17:21,693 INFO [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 508b45081a20fbcb4fc7da445cb3ab9f columnFamilyName A 2024-12-10T07:17:21,695 DEBUG [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:17:21,696 INFO [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] regionserver.HStore(327): Store=508b45081a20fbcb4fc7da445cb3ab9f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:17:21,697 INFO [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:21,697 INFO [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:17:21,698 INFO [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 508b45081a20fbcb4fc7da445cb3ab9f columnFamilyName B 2024-12-10T07:17:21,698 DEBUG [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:17:21,698 INFO [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] regionserver.HStore(327): Store=508b45081a20fbcb4fc7da445cb3ab9f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:17:21,698 INFO [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:21,699 INFO [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-10T07:17:21,699 INFO [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 508b45081a20fbcb4fc7da445cb3ab9f columnFamilyName C 2024-12-10T07:17:21,699 DEBUG [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:17:21,700 INFO [StoreOpener-508b45081a20fbcb4fc7da445cb3ab9f-1 {}] regionserver.HStore(327): Store=508b45081a20fbcb4fc7da445cb3ab9f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T07:17:21,700 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:21,700 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1038): replaying wal for 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:21,701 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:21,702 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:21,703 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1048): stopping wal replay for 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:21,703 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1060): Cleaning up temporary data for 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:21,703 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T07:17:21,705 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1093): writing seq id for 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:21,705 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1114): Opened 508b45081a20fbcb4fc7da445cb3ab9f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61842601, jitterRate=-0.07847343385219574}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T07:17:21,706 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:21,706 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegion(1006): Region open journal for 508b45081a20fbcb4fc7da445cb3ab9f: Running coprocessor pre-open hook at 1733815041686Writing region info on filesystem at 1733815041686Initializing all the Stores at 1733815041687 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733815041687Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733815041692 (+5 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733815041692Cleaning up temporary data from old regions at 1733815041703 (+11 ms)Running coprocessor post-open hooks at 1733815041706 (+3 ms)Region opened successfully at 1733815041706 2024-12-10T07:17:21,707 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., pid=131, masterSystemTime=1733815041682 2024-12-10T07:17:21,709 DEBUG [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:21,709 INFO [RS_OPEN_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_OPEN_REGION, pid=131}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:21,710 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=508b45081a20fbcb4fc7da445cb3ab9f, regionState=OPEN, openSeqNum=5, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:21,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=131, ppid=129, state=RUNNABLE, hasLock=false; OpenRegionProcedure 508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 because future has completed 2024-12-10T07:17:21,715 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=131, resume processing ppid=129 2024-12-10T07:17:21,715 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, ppid=129, state=SUCCESS, hasLock=false; OpenRegionProcedure 508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 in 183 msec 2024-12-10T07:17:21,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-12-10T07:17:21,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=508b45081a20fbcb4fc7da445cb3ab9f, REOPEN/MOVE in 507 msec 2024-12-10T07:17:21,720 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-10T07:17:21,721 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 512 msec 2024-12-10T07:17:21,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 533 msec 2024-12-10T07:17:21,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-10T07:17:21,725 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:17:21,726 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d5ba127, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,726 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:17:21,726 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:17:21,727 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:17:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:17:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:17:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a13280e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:17:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:17:21,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:21,728 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:17:21,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4099170d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,729 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@110f2d48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,729 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:17:21,729 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:17:21,730 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:17:21,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:17:21,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:17:21,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36de21ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:17:21,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:17:21,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:21,732 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:17:21,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c3f3296, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,733 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cbb80ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,733 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:17:21,733 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:17:21,734 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:17:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:17:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:17:21,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@346db817, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:17:21,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:17:21,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:21,736 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:17:21,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c82089e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,738 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50101ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,738 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:17:21,738 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:17:21,739 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:17:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:17:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:17:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53780b8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:17:21,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:17:21,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:21,741 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:17:21,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@693a6e6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e7631c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,743 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:17:21,743 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:17:21,744 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:17:21,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:17:21,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:17:21,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6996123a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:17:21,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:17:21,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:21,750 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:17:21,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4892413b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,752 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55c3cfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,752 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:17:21,752 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:17:21,753 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:17:21,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:17:21,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:17:21,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7813b8d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:17:21,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:17:21,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:21,754 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:17:21,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b7209ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58b067c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,755 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:17:21,755 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:17:21,756 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:17:21,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:17:21,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:17:21,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31cf05f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:17:21,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:17:21,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:21,757 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:17:21,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1425a197, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ebeeab1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,759 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:17:21,759 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:17:21,760 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:17:21,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:17:21,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:17:21,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59e4d588, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:17:21,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:17:21,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:21,762 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:17:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ae2200f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,765 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4de9ea07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,765 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:17:21,765 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:17:21,766 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:17:21,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:17:21,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:17:21,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20b8534d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:17:21,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:17:21,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:21,767 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:17:21,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8e8df5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,768 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff203bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,768 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b1396b8d2d12,36259,-1 for getting cluster id 2024-12-10T07:17:21,768 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T07:17:21,769 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff9dec8b-7c7e-4ce8-ab85-50b35520afdc' 2024-12-10T07:17:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T07:17:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff9dec8b-7c7e-4ce8-ab85-50b35520afdc" 2024-12-10T07:17:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68d5860e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b1396b8d2d12,36259,-1] 2024-12-10T07:17:21,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T07:17:21,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:21,771 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T07:17:21,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2557c061, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T07:17:21,772 DEBUG [Thread-2244 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:17:21,772 DEBUG [Thread-2252 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:17:21,773 DEBUG [Thread-2257 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:17:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:17:21,773 DEBUG [Thread-2259 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:17:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:17:21,773 DEBUG [Thread-2261 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:17:21,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:17:21,774 DEBUG [Thread-2259 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:17:21,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:17:21,774 DEBUG [Thread-2259 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:17:21,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:17:21,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:17:21,775 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:17:21,777 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:17:21,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:17:21,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:17:21,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:17:21,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:17:21,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:17:21,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:17:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:17:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:17:21,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:17:21,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:17:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:21,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 508b45081a20fbcb4fc7da445cb3ab9f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-10T07:17:21,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=A 2024-12-10T07:17:21,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:21,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=B 2024-12-10T07:17:21,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:21,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=C 2024-12-10T07:17:21,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:21,805 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:17:21,805 DEBUG [Thread-2263 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:17:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:17:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=132, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees 2024-12-10T07:17:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:17:21,807 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:17:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-12-10T07:17:21,809 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=132, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:17:21,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:17:21,809 DEBUG [Thread-2255 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:17:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:17:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:17:21,811 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=132, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:17:21,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:17:21,812 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:17:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:17:21,824 DEBUG [Thread-2246 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:17:21,824 DEBUG [Thread-2250 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:17:21,825 DEBUG [Thread-2250 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:17:21,825 DEBUG [Thread-2250 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:17:21,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:17:21,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:17:21,827 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:17:21,828 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:17:21,833 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:17:21,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:17:21,850 DEBUG [Thread-2248 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T07:17:21,852 DEBUG [Thread-2248 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b1396b8d2d12,35871,1733814875835, seqNum=-1] 2024-12-10T07:17:21,852 DEBUG [Thread-2248 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T07:17:21,853 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T07:17:21,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5] 2024-12-10T07:17:21,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121012f9e6c30cba499888e184f7111d8b5b_508b45081a20fbcb4fc7da445cb3ab9f is 50, key is test_row_1/A:col10/1733815041791/Put/seqid=0 2024-12-10T07:17:21,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733815041848, max timestamp=1733815041863Num uniques 859; 2024-12-10T07:17:21,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742322_1498 (size=9714) 2024-12-10T07:17:21,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T07:17:21,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733815041848, max timestamp=1733815041863Num uniques 859; 2024-12-10T07:17:21,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-10T07:17:21,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733815041848, max timestamp=1733815041863Num uniques 859; 2024-12-10T07:17:21,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-10T07:17:21,895 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:17:21,901 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121012f9e6c30cba499888e184f7111d8b5b_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121012f9e6c30cba499888e184f7111d8b5b_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:21,903 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/ebb91e37f7724268a25519c690bd8e5a, store: [table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:21,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/ebb91e37f7724268a25519c690bd8e5a is 175, key is test_row_1/A:col10/1733815041791/Put/seqid=0 2024-12-10T07:17:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-12-10T07:17:21,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56968 deadline: 1733815101918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:21,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:21,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56978 deadline: 1733815101917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:21,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:21,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56902 deadline: 1733815101923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:21,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56910 deadline: 1733815101923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:21,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815101928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:21,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:21,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:21,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742323_1499 (size=22361) 2024-12-10T07:17:21,957 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/ebb91e37f7724268a25519c690bd8e5a 2024-12-10T07:17:21,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-12-10T07:17:21,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:21,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:21,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:21,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:21,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:22,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/497c12409fcb4686bba222a77616a443 is 50, key is test_row_1/B:col10/1733815041791/Put/seqid=0 2024-12-10T07:17:22,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56978 deadline: 1733815102035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:22,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:22,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56910 deadline: 1733815102035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56968 deadline: 1733815102035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:22,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:22,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56902 deadline: 1733815102035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:22,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:22,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:22,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:22,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815102045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:22,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742324_1500 (size=9657) 2024-12-10T07:17:22,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/497c12409fcb4686bba222a77616a443 2024-12-10T07:17:22,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/0177acb73605464cb66e20e00432ab99 is 50, key is test_row_1/C:col10/1733815041791/Put/seqid=0 2024-12-10T07:17:22,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742325_1501 (size=9657) 2024-12-10T07:17:22,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/0177acb73605464cb66e20e00432ab99 2024-12-10T07:17:22,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/ebb91e37f7724268a25519c690bd8e5a as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/ebb91e37f7724268a25519c690bd8e5a 2024-12-10T07:17:22,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/ebb91e37f7724268a25519c690bd8e5a, entries=100, sequenceid=14, filesize=21.8 K 2024-12-10T07:17:22,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/497c12409fcb4686bba222a77616a443 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/497c12409fcb4686bba222a77616a443 2024-12-10T07:17:22,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/497c12409fcb4686bba222a77616a443, entries=100, sequenceid=14, filesize=9.4 K 2024-12-10T07:17:22,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/0177acb73605464cb66e20e00432ab99 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/0177acb73605464cb66e20e00432ab99 2024-12-10T07:17:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-12-10T07:17:22,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/0177acb73605464cb66e20e00432ab99, entries=100, sequenceid=14, filesize=9.4 K 2024-12-10T07:17:22,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.39 KB/136590 for 508b45081a20fbcb4fc7da445cb3ab9f in 327ms, sequenceid=14, compaction requested=false 2024-12-10T07:17:22,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:22,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-12-10T07:17:22,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:22,128 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2902): Flushing 508b45081a20fbcb4fc7da445cb3ab9f 3/3 column families, dataSize=133.39 KB heapSize=383.16 KB 2024-12-10T07:17:22,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=A 2024-12-10T07:17:22,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:17:22,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=B 2024-12-10T07:17:22,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:17:22,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=C 2024-12-10T07:17:22,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:17:22,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121085826227ee60480b967b9518b4dd5795_508b45081a20fbcb4fc7da445cb3ab9f is 50, key is test_row_0/A:col10/1733815041915/Put/seqid=0 2024-12-10T07:17:22,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742326_1502 (size=12154) 2024-12-10T07:17:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:22,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:22,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56978 deadline: 1733815102250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:22,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:22,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56968 deadline: 1733815102252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:22,256 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,256 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,256 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:22,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:22,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56902 deadline: 1733815102255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:22,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:22,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815102255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:22,258 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:22,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56910 deadline: 1733815102258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:22,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:22,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56968 deadline: 1733815102364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:22,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:22,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56978 deadline: 1733815102364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:22,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:22,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:22,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56910 deadline: 1733815102364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:22,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:22,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:22,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56902 deadline: 1733815102364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:22,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-12-10T07:17:22,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:17:22,547 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121085826227ee60480b967b9518b4dd5795_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121085826227ee60480b967b9518b4dd5795_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:22,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/24caff50adb34b4bb863d3f1e49f9ca7, store: [table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:22,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/24caff50adb34b4bb863d3f1e49f9ca7 is 175, key is test_row_0/A:col10/1733815041915/Put/seqid=0 2024-12-10T07:17:22,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815102564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:22,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56968 deadline: 1733815102575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:22,576 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56910 deadline: 1733815102576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56978 deadline: 1733815102576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:22,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56902 deadline: 1733815102577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:22,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:22,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742327_1503 (size=30955) 2024-12-10T07:17:22,593 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/24caff50adb34b4bb863d3f1e49f9ca7 2024-12-10T07:17:22,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/d122700271a248e9bb9fc39820ff59f5 is 50, key is test_row_0/B:col10/1733815041915/Put/seqid=0 2024-12-10T07:17:22,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742328_1504 (size=12001) 2024-12-10T07:17:22,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:22,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56968 deadline: 1733815102884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56910 deadline: 1733815102884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:22,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:22,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:22,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56902 deadline: 1733815102884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:22,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56978 deadline: 1733815102884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:22,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:22,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:22,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:22,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:22,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-12-10T07:17:23,008 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/d122700271a248e9bb9fc39820ff59f5 2024-12-10T07:17:23,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/d1e33186366942738716ab2fae75a112 is 50, key is test_row_0/C:col10/1733815041915/Put/seqid=0 2024-12-10T07:17:23,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742329_1505 (size=12001) 2024-12-10T07:17:23,017 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/d1e33186366942738716ab2fae75a112 2024-12-10T07:17:23,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/24caff50adb34b4bb863d3f1e49f9ca7 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/24caff50adb34b4bb863d3f1e49f9ca7 2024-12-10T07:17:23,024 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/24caff50adb34b4bb863d3f1e49f9ca7, entries=150, sequenceid=51, filesize=30.2 K 2024-12-10T07:17:23,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/d122700271a248e9bb9fc39820ff59f5 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/d122700271a248e9bb9fc39820ff59f5 2024-12-10T07:17:23,028 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/d122700271a248e9bb9fc39820ff59f5, entries=150, sequenceid=51, filesize=11.7 K 2024-12-10T07:17:23,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/d1e33186366942738716ab2fae75a112 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/d1e33186366942738716ab2fae75a112 2024-12-10T07:17:23,031 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/d1e33186366942738716ab2fae75a112, entries=150, sequenceid=51, filesize=11.7 K 2024-12-10T07:17:23,032 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(3140): Finished flush of dataSize ~133.39 KB/136590, heapSize ~383.11 KB/392304, currentSize=53.67 KB/54960 for 508b45081a20fbcb4fc7da445cb3ab9f in 904ms, sequenceid=51, compaction requested=false 2024-12-10T07:17:23,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2603): Flush status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:23,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:23,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=133 2024-12-10T07:17:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=133 2024-12-10T07:17:23,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-12-10T07:17:23,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2220 sec 2024-12-10T07:17:23,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees in 1.2300 sec 2024-12-10T07:17:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:23,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 508b45081a20fbcb4fc7da445cb3ab9f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-10T07:17:23,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=A 2024-12-10T07:17:23,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:23,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=B 2024-12-10T07:17:23,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:23,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=C 2024-12-10T07:17:23,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:23,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121069f7238198594e5b9b0776bf31934735_508b45081a20fbcb4fc7da445cb3ab9f is 50, key is test_row_0/A:col10/1733815042247/Put/seqid=0 2024-12-10T07:17:23,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742330_1506 (size=14594) 2024-12-10T07:17:23,089 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:17:23,095 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121069f7238198594e5b9b0776bf31934735_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121069f7238198594e5b9b0776bf31934735_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:23,097 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/7037f436352f4b6ab207a7c5c871ac2c, store: [table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:23,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/7037f436352f4b6ab207a7c5c871ac2c is 175, key is test_row_0/A:col10/1733815042247/Put/seqid=0 2024-12-10T07:17:23,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742331_1507 (size=39549) 2024-12-10T07:17:23,109 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/7037f436352f4b6ab207a7c5c871ac2c 2024-12-10T07:17:23,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/88161cde38c446b894043ecdecdc1989 is 50, key is test_row_0/B:col10/1733815042247/Put/seqid=0 2024-12-10T07:17:23,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742332_1508 (size=12001) 2024-12-10T07:17:23,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815103150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:23,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815103255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:23,258 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:23,258 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:23,258 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:23,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56978 deadline: 1733815103394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:23,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56902 deadline: 1733815103394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:23,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56968 deadline: 1733815103394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:23,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56910 deadline: 1733815103395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:23,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815103464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:23,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:23,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:23,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:23,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/88161cde38c446b894043ecdecdc1989 2024-12-10T07:17:23,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/336939c37f9749aea7693987327fb036 is 50, key is test_row_0/C:col10/1733815042247/Put/seqid=0 2024-12-10T07:17:23,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742333_1509 (size=12001) 2024-12-10T07:17:23,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/336939c37f9749aea7693987327fb036 2024-12-10T07:17:23,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/7037f436352f4b6ab207a7c5c871ac2c as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/7037f436352f4b6ab207a7c5c871ac2c 2024-12-10T07:17:23,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/7037f436352f4b6ab207a7c5c871ac2c, entries=200, sequenceid=63, filesize=38.6 K 2024-12-10T07:17:23,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/88161cde38c446b894043ecdecdc1989 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/88161cde38c446b894043ecdecdc1989 2024-12-10T07:17:23,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/88161cde38c446b894043ecdecdc1989, entries=150, sequenceid=63, filesize=11.7 K 2024-12-10T07:17:23,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/336939c37f9749aea7693987327fb036 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/336939c37f9749aea7693987327fb036 2024-12-10T07:17:23,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/336939c37f9749aea7693987327fb036, entries=150, sequenceid=63, filesize=11.7 K 2024-12-10T07:17:23,575 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 508b45081a20fbcb4fc7da445cb3ab9f in 499ms, sequenceid=63, compaction requested=true 2024-12-10T07:17:23,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:23,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 508b45081a20fbcb4fc7da445cb3ab9f:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:17:23,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:17:23,575 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:17:23,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 508b45081a20fbcb4fc7da445cb3ab9f:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:17:23,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:17:23,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 508b45081a20fbcb4fc7da445cb3ab9f:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:17:23,575 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:17:23,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:17:23,576 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:17:23,576 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 508b45081a20fbcb4fc7da445cb3ab9f/A is initiating minor compaction (all files) 2024-12-10T07:17:23,576 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 508b45081a20fbcb4fc7da445cb3ab9f/A in TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:23,576 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/ebb91e37f7724268a25519c690bd8e5a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/24caff50adb34b4bb863d3f1e49f9ca7, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/7037f436352f4b6ab207a7c5c871ac2c] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp, totalSize=90.7 K 2024-12-10T07:17:23,576 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:23,576 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. files: [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/ebb91e37f7724268a25519c690bd8e5a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/24caff50adb34b4bb863d3f1e49f9ca7, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/7037f436352f4b6ab207a7c5c871ac2c] 2024-12-10T07:17:23,577 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:17:23,577 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 508b45081a20fbcb4fc7da445cb3ab9f/B is initiating minor compaction (all files) 2024-12-10T07:17:23,577 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 508b45081a20fbcb4fc7da445cb3ab9f/B in TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:23,577 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/497c12409fcb4686bba222a77616a443, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/d122700271a248e9bb9fc39820ff59f5, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/88161cde38c446b894043ecdecdc1989] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp, totalSize=32.9 K 2024-12-10T07:17:23,577 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebb91e37f7724268a25519c690bd8e5a, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733815041791 2024-12-10T07:17:23,578 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 497c12409fcb4686bba222a77616a443, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733815041791 2024-12-10T07:17:23,578 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24caff50adb34b4bb863d3f1e49f9ca7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733815041884 2024-12-10T07:17:23,578 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting d122700271a248e9bb9fc39820ff59f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733815041884 2024-12-10T07:17:23,578 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7037f436352f4b6ab207a7c5c871ac2c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733815042247 2024-12-10T07:17:23,578 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 88161cde38c446b894043ecdecdc1989, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733815042247 2024-12-10T07:17:23,586 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:23,593 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210d41c348b57964da4a4a3bb0678c3232d_508b45081a20fbcb4fc7da445cb3ab9f store=[table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:23,594 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210d41c348b57964da4a4a3bb0678c3232d_508b45081a20fbcb4fc7da445cb3ab9f, store=[table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:23,595 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210d41c348b57964da4a4a3bb0678c3232d_508b45081a20fbcb4fc7da445cb3ab9f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:23,605 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 508b45081a20fbcb4fc7da445cb3ab9f#B#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:17:23,605 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/c52c416596994fb59bb59526fe6a9e84 is 50, key is test_row_0/B:col10/1733815042247/Put/seqid=0 2024-12-10T07:17:23,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742334_1510 (size=4469) 2024-12-10T07:17:23,632 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 508b45081a20fbcb4fc7da445cb3ab9f#A#compaction#422 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:17:23,633 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/77a6d2ffe81d41cea6c0f087e8ee3f2d is 175, key is test_row_0/A:col10/1733815042247/Put/seqid=0 2024-12-10T07:17:23,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742335_1511 (size=12104) 2024-12-10T07:17:23,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742336_1512 (size=31058) 2024-12-10T07:17:23,754 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T07:17:23,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733815043125, max timestamp=1733815043777Num uniques 859; 2024-12-10T07:17:23,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/b1396b8d2d12:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776  suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-10T07:17:23,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 508b45081a20fbcb4fc7da445cb3ab9f 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-12-10T07:17:23,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=A 2024-12-10T07:17:23,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:17:23,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=B 2024-12-10T07:17:23,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:17:23,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=C 2024-12-10T07:17:23,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-10T07:17:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:23,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210ee6c6e2057854beebe280f4021f47d58_508b45081a20fbcb4fc7da445cb3ab9f is 50, key is test_row_0/A:col10/1733815043145/Put/seqid=0 2024-12-10T07:17:23,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742337_1513 (size=14594) 2024-12-10T07:17:23,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815103944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:23,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:23,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:23,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-12-10T07:17:23,954 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:17:23,955 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:17:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=134, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees 2024-12-10T07:17:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-10T07:17:23,957 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=134, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:17:23,959 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=134, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:17:23,959 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:17:24,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815104055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:24,061 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/c52c416596994fb59bb59526fe6a9e84 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/c52c416596994fb59bb59526fe6a9e84 2024-12-10T07:17:24,064 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 508b45081a20fbcb4fc7da445cb3ab9f/B of 508b45081a20fbcb4fc7da445cb3ab9f into c52c416596994fb59bb59526fe6a9e84(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:17:24,064 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:24,064 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., storeName=508b45081a20fbcb4fc7da445cb3ab9f/B, priority=13, startTime=1733815043575; duration=0sec 2024-12-10T07:17:24,064 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:17:24,064 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 508b45081a20fbcb4fc7da445cb3ab9f:B 2024-12-10T07:17:24,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-10T07:17:24,064 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:17:24,065 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:17:24,065 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 508b45081a20fbcb4fc7da445cb3ab9f/C is initiating minor compaction (all files) 2024-12-10T07:17:24,065 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 508b45081a20fbcb4fc7da445cb3ab9f/C in TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:24,065 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/0177acb73605464cb66e20e00432ab99, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/d1e33186366942738716ab2fae75a112, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/336939c37f9749aea7693987327fb036] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp, totalSize=32.9 K 2024-12-10T07:17:24,065 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 0177acb73605464cb66e20e00432ab99, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733815041791 2024-12-10T07:17:24,066 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting d1e33186366942738716ab2fae75a112, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733815041884 2024-12-10T07:17:24,066 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 336939c37f9749aea7693987327fb036, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733815042247 2024-12-10T07:17:24,071 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 508b45081a20fbcb4fc7da445cb3ab9f#C#compaction#425 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:17:24,071 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/777738d2ece34da29944a57899339cdc is 50, key is test_row_0/C:col10/1733815042247/Put/seqid=0 2024-12-10T07:17:24,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742338_1514 (size=12104) 2024-12-10T07:17:24,088 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/77a6d2ffe81d41cea6c0f087e8ee3f2d as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/77a6d2ffe81d41cea6c0f087e8ee3f2d 2024-12-10T07:17:24,091 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 508b45081a20fbcb4fc7da445cb3ab9f/A of 508b45081a20fbcb4fc7da445cb3ab9f into 77a6d2ffe81d41cea6c0f087e8ee3f2d(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:17:24,091 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:24,091 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., storeName=508b45081a20fbcb4fc7da445cb3ab9f/A, priority=13, startTime=1733815043575; duration=0sec 2024-12-10T07:17:24,092 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:17:24,092 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 508b45081a20fbcb4fc7da445cb3ab9f:A 2024-12-10T07:17:24,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-12-10T07:17:24,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:24,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:24,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:24,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:24,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:24,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-12-10T07:17:24,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:24,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:24,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:24,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:24,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:24,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815104265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:24,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-10T07:17:24,283 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:17:24,285 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210ee6c6e2057854beebe280f4021f47d58_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ee6c6e2057854beebe280f4021f47d58_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:24,286 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/1560e408f5cc456fb93c58503613a1bd, store: [table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:24,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/1560e408f5cc456fb93c58503613a1bd is 175, key is test_row_0/A:col10/1733815043145/Put/seqid=0 2024-12-10T07:17:24,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742339_1515 (size=39549) 2024-12-10T07:17:24,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:24,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56902 deadline: 1733815104404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56978 deadline: 1733815104404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:24,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56968 deadline: 1733815104404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:24,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56910 deadline: 1733815104405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:24,407 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_0' in 508b45081a20fbcb4fc7da445cb3ab9f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:17:24,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:24,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:24,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:24,407 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_0' in 508b45081a20fbcb4fc7da445cb3ab9f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:17:24,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:24,407 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_2' in 508b45081a20fbcb4fc7da445cb3ab9f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:17:24,407 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:24,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:24,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:24,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:24,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:24,407 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 508b45081a20fbcb4fc7da445cb3ab9f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:17:24,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:24,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:24,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:24,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-12-10T07:17:24,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:24,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:24,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:24,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:24,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:24,485 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/777738d2ece34da29944a57899339cdc as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/777738d2ece34da29944a57899339cdc 2024-12-10T07:17:24,490 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 508b45081a20fbcb4fc7da445cb3ab9f/C of 508b45081a20fbcb4fc7da445cb3ab9f into 777738d2ece34da29944a57899339cdc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:17:24,490 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:24,490 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., storeName=508b45081a20fbcb4fc7da445cb3ab9f/C, priority=13, startTime=1733815043575; duration=0sec 2024-12-10T07:17:24,491 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:17:24,491 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 508b45081a20fbcb4fc7da445cb3ab9f:C 2024-12-10T07:17:24,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-12-10T07:17:24,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:24,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:24,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:24,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:24,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:24,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815104575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-10T07:17:24,690 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/1560e408f5cc456fb93c58503613a1bd 2024-12-10T07:17:24,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/57799e2bdcbd445ba8ba8c7ebad7f838 is 50, key is test_row_0/B:col10/1733815043145/Put/seqid=0 2024-12-10T07:17:24,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742340_1516 (size=12001) 2024-12-10T07:17:24,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-12-10T07:17:24,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:24,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:24,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:24,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:24,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:24,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-12-10T07:17:24,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:24,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:24,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:24,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:24,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:25,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-12-10T07:17:25,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:25,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:25,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:25,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:25,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:25,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:25,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:25,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815105085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:25,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:25,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:25,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:25,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-10T07:17:25,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/57799e2bdcbd445ba8ba8c7ebad7f838 2024-12-10T07:17:25,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/02615d2dc3494a36baacf35323071362 is 50, key is test_row_0/C:col10/1733815043145/Put/seqid=0 2024-12-10T07:17:25,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742341_1517 (size=12001) 2024-12-10T07:17:25,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-12-10T07:17:25,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:25,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:25,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:25,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:25,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:25,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-12-10T07:17:25,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:25,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:25,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:25,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:25,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:25,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:25,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-12-10T07:17:25,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:25,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:25,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:25,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(58): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:25,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:25,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/02615d2dc3494a36baacf35323071362 2024-12-10T07:17:25,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/1560e408f5cc456fb93c58503613a1bd as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/1560e408f5cc456fb93c58503613a1bd 2024-12-10T07:17:25,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/1560e408f5cc456fb93c58503613a1bd, entries=200, sequenceid=84, filesize=38.6 K 2024-12-10T07:17:25,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/57799e2bdcbd445ba8ba8c7ebad7f838 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/57799e2bdcbd445ba8ba8c7ebad7f838 2024-12-10T07:17:25,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/57799e2bdcbd445ba8ba8c7ebad7f838, entries=150, sequenceid=84, filesize=11.7 K 2024-12-10T07:17:25,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/02615d2dc3494a36baacf35323071362 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/02615d2dc3494a36baacf35323071362 2024-12-10T07:17:25,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/02615d2dc3494a36baacf35323071362, entries=150, sequenceid=84, filesize=11.7 K 2024-12-10T07:17:25,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for 508b45081a20fbcb4fc7da445cb3ab9f in 1809ms, sequenceid=84, compaction requested=false 2024-12-10T07:17:25,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:25,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-12-10T07:17:25,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:25,645 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2902): Flushing 508b45081a20fbcb4fc7da445cb3ab9f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T07:17:25,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=A 2024-12-10T07:17:25,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:25,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=B 2024-12-10T07:17:25,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:25,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=C 2024-12-10T07:17:25,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:25,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121076b3183adb38422ab28cf5938f40a191_508b45081a20fbcb4fc7da445cb3ab9f is 50, key is test_row_0/A:col10/1733815043941/Put/seqid=0 2024-12-10T07:17:25,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742342_1518 (size=12154) 2024-12-10T07:17:26,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:17:26,082 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121076b3183adb38422ab28cf5938f40a191_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121076b3183adb38422ab28cf5938f40a191_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:26,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/06976589c1e94a3a9a9c4a07ade08afa, store: [table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:26,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/06976589c1e94a3a9a9c4a07ade08afa is 175, key is test_row_0/A:col10/1733815043941/Put/seqid=0 2024-12-10T07:17:26,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:26,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742343_1519 (size=30955) 2024-12-10T07:17:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-10T07:17:26,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815106140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:26,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815106254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:26,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:26,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:26,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:26,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56978 deadline: 1733815106415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:26,417 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 508b45081a20fbcb4fc7da445cb3ab9f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:17:26,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:26,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:26,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:26,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56902 deadline: 1733815106434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:26,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56968 deadline: 1733815106434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:26,437 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_0' in 508b45081a20fbcb4fc7da445cb3ab9f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:17:26,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:26,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:26,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:26,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56910 deadline: 1733815106434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:26,441 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_0' in 508b45081a20fbcb4fc7da445cb3ab9f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:17:26,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:26,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:26,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:26,441 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_2' in 508b45081a20fbcb4fc7da445cb3ab9f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:17:26,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:26,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:26,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:26,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815106465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:26,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:26,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:26,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:26,499 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=103, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/06976589c1e94a3a9a9c4a07ade08afa 2024-12-10T07:17:26,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/277a36a6d8f440469a44d95c3d8192fa is 50, key is test_row_0/B:col10/1733815043941/Put/seqid=0 2024-12-10T07:17:26,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742344_1520 (size=12001) 2024-12-10T07:17:26,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815106775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:26,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:26,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:26,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:26,908 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/277a36a6d8f440469a44d95c3d8192fa 2024-12-10T07:17:26,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/0ebfa66481e043ca89ded21247cfb1ea is 50, key is test_row_0/C:col10/1733815043941/Put/seqid=0 2024-12-10T07:17:26,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742345_1521 (size=12001) 2024-12-10T07:17:26,985 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/0ebfa66481e043ca89ded21247cfb1ea 2024-12-10T07:17:26,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/06976589c1e94a3a9a9c4a07ade08afa as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/06976589c1e94a3a9a9c4a07ade08afa 2024-12-10T07:17:26,997 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/06976589c1e94a3a9a9c4a07ade08afa, entries=150, sequenceid=103, filesize=30.2 K 2024-12-10T07:17:26,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/277a36a6d8f440469a44d95c3d8192fa as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/277a36a6d8f440469a44d95c3d8192fa 2024-12-10T07:17:27,002 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/277a36a6d8f440469a44d95c3d8192fa, entries=150, sequenceid=103, filesize=11.7 K 2024-12-10T07:17:27,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/0ebfa66481e043ca89ded21247cfb1ea as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/0ebfa66481e043ca89ded21247cfb1ea 2024-12-10T07:17:27,007 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/0ebfa66481e043ca89ded21247cfb1ea, entries=150, sequenceid=103, filesize=11.7 K 2024-12-10T07:17:27,009 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 508b45081a20fbcb4fc7da445cb3ab9f in 1364ms, sequenceid=103, compaction requested=true 2024-12-10T07:17:27,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2603): Flush status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:27,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:27,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=135 2024-12-10T07:17:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=135 2024-12-10T07:17:27,014 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-12-10T07:17:27,014 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0530 sec 2024-12-10T07:17:27,016 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees in 3.0600 sec 2024-12-10T07:17:27,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:27,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 508b45081a20fbcb4fc7da445cb3ab9f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T07:17:27,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=A 2024-12-10T07:17:27,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:27,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=B 2024-12-10T07:17:27,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:27,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=C 2024-12-10T07:17:27,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:27,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a7f663e7cbcb4d6a8cc2e1c1496661cb_508b45081a20fbcb4fc7da445cb3ab9f is 50, key is test_row_0/A:col10/1733815046136/Put/seqid=0 2024-12-10T07:17:27,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742346_1522 (size=14594) 2024-12-10T07:17:27,336 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:17:27,338 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a7f663e7cbcb4d6a8cc2e1c1496661cb_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a7f663e7cbcb4d6a8cc2e1c1496661cb_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:27,339 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/f132873524784f45a75c24504aea39e7, store: [table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:27,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/f132873524784f45a75c24504aea39e7 is 175, key is test_row_0/A:col10/1733815046136/Put/seqid=0 2024-12-10T07:17:27,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742347_1523 (size=39549) 2024-12-10T07:17:27,377 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/f132873524784f45a75c24504aea39e7 2024-12-10T07:17:27,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/551547d9a9cf40f6b26a9dceb6711052 is 50, key is test_row_0/B:col10/1733815046136/Put/seqid=0 2024-12-10T07:17:27,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815107401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:27,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:27,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:27,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:27,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742348_1524 (size=12001) 2024-12-10T07:17:27,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:27,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815107517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:27,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:27,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:27,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:27,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815107724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:27,727 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:27,727 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:27,727 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:27,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/551547d9a9cf40f6b26a9dceb6711052 2024-12-10T07:17:27,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/406ba00dcbbc439185f24d05b534d4af is 50, key is test_row_0/C:col10/1733815046136/Put/seqid=0 2024-12-10T07:17:27,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742349_1525 (size=12001) 2024-12-10T07:17:27,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/406ba00dcbbc439185f24d05b534d4af 2024-12-10T07:17:27,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/f132873524784f45a75c24504aea39e7 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/f132873524784f45a75c24504aea39e7 2024-12-10T07:17:27,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/f132873524784f45a75c24504aea39e7, entries=200, sequenceid=120, filesize=38.6 K 2024-12-10T07:17:27,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/551547d9a9cf40f6b26a9dceb6711052 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/551547d9a9cf40f6b26a9dceb6711052 2024-12-10T07:17:27,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/551547d9a9cf40f6b26a9dceb6711052, entries=150, sequenceid=120, filesize=11.7 K 2024-12-10T07:17:27,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/406ba00dcbbc439185f24d05b534d4af as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/406ba00dcbbc439185f24d05b534d4af 2024-12-10T07:17:27,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/406ba00dcbbc439185f24d05b534d4af, entries=150, sequenceid=120, filesize=11.7 K 2024-12-10T07:17:27,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 508b45081a20fbcb4fc7da445cb3ab9f in 578ms, sequenceid=120, compaction requested=true 2024-12-10T07:17:27,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:27,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 508b45081a20fbcb4fc7da445cb3ab9f:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:17:27,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:17:27,868 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:17:27,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 508b45081a20fbcb4fc7da445cb3ab9f:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:17:27,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:17:27,868 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:17:27,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 508b45081a20fbcb4fc7da445cb3ab9f:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:17:27,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:17:27,869 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:17:27,869 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 508b45081a20fbcb4fc7da445cb3ab9f/B is initiating minor compaction (all files) 2024-12-10T07:17:27,870 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 508b45081a20fbcb4fc7da445cb3ab9f/B in TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:27,870 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/c52c416596994fb59bb59526fe6a9e84, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/57799e2bdcbd445ba8ba8c7ebad7f838, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/277a36a6d8f440469a44d95c3d8192fa, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/551547d9a9cf40f6b26a9dceb6711052] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp, totalSize=47.0 K 2024-12-10T07:17:27,871 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:17:27,871 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting c52c416596994fb59bb59526fe6a9e84, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733815042247 2024-12-10T07:17:27,871 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 508b45081a20fbcb4fc7da445cb3ab9f/A is initiating minor compaction (all files) 2024-12-10T07:17:27,871 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 508b45081a20fbcb4fc7da445cb3ab9f/A in TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:27,871 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/77a6d2ffe81d41cea6c0f087e8ee3f2d, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/1560e408f5cc456fb93c58503613a1bd, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/06976589c1e94a3a9a9c4a07ade08afa, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/f132873524784f45a75c24504aea39e7] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp, totalSize=137.8 K 2024-12-10T07:17:27,871 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 57799e2bdcbd445ba8ba8c7ebad7f838, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733815043125 2024-12-10T07:17:27,871 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:27,872 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. files: [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/77a6d2ffe81d41cea6c0f087e8ee3f2d, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/1560e408f5cc456fb93c58503613a1bd, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/06976589c1e94a3a9a9c4a07ade08afa, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/f132873524784f45a75c24504aea39e7] 2024-12-10T07:17:27,872 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77a6d2ffe81d41cea6c0f087e8ee3f2d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733815042247 2024-12-10T07:17:27,872 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 277a36a6d8f440469a44d95c3d8192fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733815043897 2024-12-10T07:17:27,873 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1560e408f5cc456fb93c58503613a1bd, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733815043125 2024-12-10T07:17:27,873 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 551547d9a9cf40f6b26a9dceb6711052, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733815046136 2024-12-10T07:17:27,876 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06976589c1e94a3a9a9c4a07ade08afa, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733815043897 2024-12-10T07:17:27,877 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting f132873524784f45a75c24504aea39e7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733815046132 2024-12-10T07:17:27,891 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:27,892 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 508b45081a20fbcb4fc7da445cb3ab9f#B#compaction#434 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:17:27,892 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412102efc005b96a9438d8c4e1710b7ba151d_508b45081a20fbcb4fc7da445cb3ab9f store=[table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:27,893 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/9ec1b479822f46028921949a15c595c8 is 50, key is test_row_0/B:col10/1733815046136/Put/seqid=0 2024-12-10T07:17:27,895 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412102efc005b96a9438d8c4e1710b7ba151d_508b45081a20fbcb4fc7da445cb3ab9f, store=[table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:27,895 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412102efc005b96a9438d8c4e1710b7ba151d_508b45081a20fbcb4fc7da445cb3ab9f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:27,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742350_1526 (size=12241) 2024-12-10T07:17:27,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742351_1527 (size=4469) 2024-12-10T07:17:27,905 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/9ec1b479822f46028921949a15c595c8 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/9ec1b479822f46028921949a15c595c8 2024-12-10T07:17:27,912 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 508b45081a20fbcb4fc7da445cb3ab9f/B of 508b45081a20fbcb4fc7da445cb3ab9f into 9ec1b479822f46028921949a15c595c8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:17:27,912 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:27,912 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., storeName=508b45081a20fbcb4fc7da445cb3ab9f/B, priority=12, startTime=1733815047868; duration=0sec 2024-12-10T07:17:27,912 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:17:27,912 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 508b45081a20fbcb4fc7da445cb3ab9f:B 2024-12-10T07:17:27,912 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:17:27,914 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:17:27,914 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 508b45081a20fbcb4fc7da445cb3ab9f/C is initiating minor compaction (all files) 2024-12-10T07:17:27,914 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 508b45081a20fbcb4fc7da445cb3ab9f/C in TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:27,914 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/777738d2ece34da29944a57899339cdc, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/02615d2dc3494a36baacf35323071362, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/0ebfa66481e043ca89ded21247cfb1ea, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/406ba00dcbbc439185f24d05b534d4af] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp, totalSize=47.0 K 2024-12-10T07:17:27,915 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 777738d2ece34da29944a57899339cdc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733815042247 2024-12-10T07:17:27,915 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 02615d2dc3494a36baacf35323071362, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733815043125 2024-12-10T07:17:27,915 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ebfa66481e043ca89ded21247cfb1ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733815043897 2024-12-10T07:17:27,915 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 406ba00dcbbc439185f24d05b534d4af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733815046136 2024-12-10T07:17:27,922 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 508b45081a20fbcb4fc7da445cb3ab9f#A#compaction#435 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:17:27,923 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/54d935fff63348db8d1af4cf2a54e25b is 175, key is test_row_0/A:col10/1733815046136/Put/seqid=0 2024-12-10T07:17:27,933 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 508b45081a20fbcb4fc7da445cb3ab9f#C#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:17:27,933 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/b166e3e4a3b847a0a116683bf9d0fd2c is 50, key is test_row_0/C:col10/1733815046136/Put/seqid=0 2024-12-10T07:17:27,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742352_1528 (size=31195) 2024-12-10T07:17:27,965 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/54d935fff63348db8d1af4cf2a54e25b as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/54d935fff63348db8d1af4cf2a54e25b 2024-12-10T07:17:27,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742353_1529 (size=12241) 2024-12-10T07:17:27,976 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 508b45081a20fbcb4fc7da445cb3ab9f/A of 508b45081a20fbcb4fc7da445cb3ab9f into 54d935fff63348db8d1af4cf2a54e25b(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:17:27,976 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:27,976 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., storeName=508b45081a20fbcb4fc7da445cb3ab9f/A, priority=12, startTime=1733815047867; duration=0sec 2024-12-10T07:17:27,976 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:17:27,977 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 508b45081a20fbcb4fc7da445cb3ab9f:A 2024-12-10T07:17:27,982 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/b166e3e4a3b847a0a116683bf9d0fd2c as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/b166e3e4a3b847a0a116683bf9d0fd2c 2024-12-10T07:17:27,988 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 508b45081a20fbcb4fc7da445cb3ab9f/C of 508b45081a20fbcb4fc7da445cb3ab9f into b166e3e4a3b847a0a116683bf9d0fd2c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:17:27,988 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:27,988 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., storeName=508b45081a20fbcb4fc7da445cb3ab9f/C, priority=12, startTime=1733815047868; duration=0sec 2024-12-10T07:17:27,988 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:17:27,989 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 508b45081a20fbcb4fc7da445cb3ab9f:C 2024-12-10T07:17:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:28,036 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 508b45081a20fbcb4fc7da445cb3ab9f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T07:17:28,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=A 2024-12-10T07:17:28,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:28,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=B 2024-12-10T07:17:28,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:28,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=C 2024-12-10T07:17:28,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:28,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210dacc9b25e35a42dcba4410194dbbcd36_508b45081a20fbcb4fc7da445cb3ab9f is 50, key is test_row_0/A:col10/1733815048035/Put/seqid=0 2024-12-10T07:17:28,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742354_1530 (size=14794) 2024-12-10T07:17:28,091 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:17:28,094 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210dacc9b25e35a42dcba4410194dbbcd36_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210dacc9b25e35a42dcba4410194dbbcd36_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:28,095 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/ae206cc75ce24eb58c73b11930433d0a, store: [table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:28,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/ae206cc75ce24eb58c73b11930433d0a is 175, key is test_row_0/A:col10/1733815048035/Put/seqid=0 2024-12-10T07:17:28,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815108095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:28,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:28,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:28,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:28,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742355_1531 (size=39749) 2024-12-10T07:17:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-10T07:17:28,125 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:17:28,125 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:17:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees 2024-12-10T07:17:28,128 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=136, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:17:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-10T07:17:28,130 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=136, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:17:28,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:17:28,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:28,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815108204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:28,208 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:28,208 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:28,208 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:28,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-10T07:17:28,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-12-10T07:17:28,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:28,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:28,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:28,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:28,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:28,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:28,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815108415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:28,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-12-10T07:17:28,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:28,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:28,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:28,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:28,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-10T07:17:28,518 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=139, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/ae206cc75ce24eb58c73b11930433d0a 2024-12-10T07:17:28,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/31461da1009e4abb917c8bf75f8570d2 is 50, key is test_row_0/B:col10/1733815048035/Put/seqid=0 2024-12-10T07:17:28,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742356_1532 (size=12151) 2024-12-10T07:17:28,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/31461da1009e4abb917c8bf75f8570d2 2024-12-10T07:17:28,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/32c398d58b854c288a15d4bd2f8c96ad is 50, key is test_row_0/C:col10/1733815048035/Put/seqid=0 2024-12-10T07:17:28,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742357_1533 (size=12151) 2024-12-10T07:17:28,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/32c398d58b854c288a15d4bd2f8c96ad 2024-12-10T07:17:28,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/ae206cc75ce24eb58c73b11930433d0a as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/ae206cc75ce24eb58c73b11930433d0a 2024-12-10T07:17:28,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-12-10T07:17:28,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:28,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:28,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:28,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:28,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:28,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/ae206cc75ce24eb58c73b11930433d0a, entries=200, sequenceid=139, filesize=38.8 K 2024-12-10T07:17:28,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/31461da1009e4abb917c8bf75f8570d2 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/31461da1009e4abb917c8bf75f8570d2 2024-12-10T07:17:28,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/31461da1009e4abb917c8bf75f8570d2, entries=150, sequenceid=139, filesize=11.9 K 2024-12-10T07:17:28,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/32c398d58b854c288a15d4bd2f8c96ad as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/32c398d58b854c288a15d4bd2f8c96ad 2024-12-10T07:17:28,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/32c398d58b854c288a15d4bd2f8c96ad, entries=150, sequenceid=139, filesize=11.9 K 2024-12-10T07:17:28,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 508b45081a20fbcb4fc7da445cb3ab9f in 581ms, sequenceid=139, compaction requested=false 2024-12-10T07:17:28,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:28,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 508b45081a20fbcb4fc7da445cb3ab9f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T07:17:28,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=A 2024-12-10T07:17:28,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:28,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=B 2024-12-10T07:17:28,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:28,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=C 2024-12-10T07:17:28,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:28,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210c9edc7c3422b4a60b079dd97a2d40be6_508b45081a20fbcb4fc7da445cb3ab9f is 50, key is test_row_0/A:col10/1733815048088/Put/seqid=0 2024-12-10T07:17:28,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742358_1534 (size=12304) 2024-12-10T07:17:28,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-12-10T07:17:28,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:28,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:28,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:28,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:28,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-10T07:17:28,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815108785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:28,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:28,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:28,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:28,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815108895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:28,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-12-10T07:17:28,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:28,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:28,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:28,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:28,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:28,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:29,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-12-10T07:17:29,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:29,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:29,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:29,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:29,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:29,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815109106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:29,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:29,140 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:17:29,163 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210c9edc7c3422b4a60b079dd97a2d40be6_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210c9edc7c3422b4a60b079dd97a2d40be6_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:29,166 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/6df73088d5884ad7919604fe3b4cf96e, store: [table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:29,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/6df73088d5884ad7919604fe3b4cf96e is 175, key is test_row_0/A:col10/1733815048088/Put/seqid=0 2024-12-10T07:17:29,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742359_1535 (size=31105) 2024-12-10T07:17:29,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-12-10T07:17:29,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:29,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:29,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:29,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:29,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:29,227 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/6df73088d5884ad7919604fe3b4cf96e 2024-12-10T07:17:29,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/11a62f303dd9472685c34a1e974ef804 is 50, key is test_row_0/B:col10/1733815048088/Put/seqid=0 2024-12-10T07:17:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-10T07:17:29,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742360_1536 (size=12151) 2024-12-10T07:17:29,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/11a62f303dd9472685c34a1e974ef804 2024-12-10T07:17:29,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/3c28d16d6d554bde842159c4968d4abd is 50, key is test_row_0/C:col10/1733815048088/Put/seqid=0 2024-12-10T07:17:29,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742361_1537 (size=12151) 2024-12-10T07:17:29,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/3c28d16d6d554bde842159c4968d4abd 2024-12-10T07:17:29,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/6df73088d5884ad7919604fe3b4cf96e as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/6df73088d5884ad7919604fe3b4cf96e 2024-12-10T07:17:29,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/6df73088d5884ad7919604fe3b4cf96e, entries=150, sequenceid=156, filesize=30.4 K 2024-12-10T07:17:29,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/11a62f303dd9472685c34a1e974ef804 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/11a62f303dd9472685c34a1e974ef804 2024-12-10T07:17:29,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-12-10T07:17:29,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:29,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:29,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:29,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:29,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:29,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/11a62f303dd9472685c34a1e974ef804, entries=150, sequenceid=156, filesize=11.9 K 2024-12-10T07:17:29,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/3c28d16d6d554bde842159c4968d4abd as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/3c28d16d6d554bde842159c4968d4abd 2024-12-10T07:17:29,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/3c28d16d6d554bde842159c4968d4abd, entries=150, sequenceid=156, filesize=11.9 K 2024-12-10T07:17:29,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 508b45081a20fbcb4fc7da445cb3ab9f in 683ms, sequenceid=156, compaction requested=true 2024-12-10T07:17:29,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:29,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 508b45081a20fbcb4fc7da445cb3ab9f:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:17:29,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:17:29,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 508b45081a20fbcb4fc7da445cb3ab9f:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:17:29,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-10T07:17:29,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 508b45081a20fbcb4fc7da445cb3ab9f:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:17:29,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-10T07:17:29,410 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:17:29,410 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:17:29,411 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:17:29,411 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 508b45081a20fbcb4fc7da445cb3ab9f/A is initiating minor compaction (all files) 2024-12-10T07:17:29,411 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 508b45081a20fbcb4fc7da445cb3ab9f/A in TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:29,411 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/54d935fff63348db8d1af4cf2a54e25b, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/ae206cc75ce24eb58c73b11930433d0a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/6df73088d5884ad7919604fe3b4cf96e] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp, totalSize=99.7 K 2024-12-10T07:17:29,411 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:29,411 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. files: [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/54d935fff63348db8d1af4cf2a54e25b, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/ae206cc75ce24eb58c73b11930433d0a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/6df73088d5884ad7919604fe3b4cf96e] 2024-12-10T07:17:29,412 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:17:29,412 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 508b45081a20fbcb4fc7da445cb3ab9f/C is initiating minor compaction (all files) 2024-12-10T07:17:29,412 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 508b45081a20fbcb4fc7da445cb3ab9f/C in TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:29,412 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/b166e3e4a3b847a0a116683bf9d0fd2c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/32c398d58b854c288a15d4bd2f8c96ad, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/3c28d16d6d554bde842159c4968d4abd] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp, totalSize=35.7 K 2024-12-10T07:17:29,412 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 54d935fff63348db8d1af4cf2a54e25b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733815046136 2024-12-10T07:17:29,412 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting b166e3e4a3b847a0a116683bf9d0fd2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733815046136 2024-12-10T07:17:29,412 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting ae206cc75ce24eb58c73b11930433d0a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733815047373 2024-12-10T07:17:29,413 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32c398d58b854c288a15d4bd2f8c96ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733815047373 2024-12-10T07:17:29,413 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c28d16d6d554bde842159c4968d4abd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733815048083 2024-12-10T07:17:29,413 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 6df73088d5884ad7919604fe3b4cf96e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733815048083 2024-12-10T07:17:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:29,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 508b45081a20fbcb4fc7da445cb3ab9f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T07:17:29,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=A 2024-12-10T07:17:29,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:29,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=B 2024-12-10T07:17:29,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:29,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=C 2024-12-10T07:17:29,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:29,431 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 508b45081a20fbcb4fc7da445cb3ab9f#C#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:17:29,432 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/db895434e3ee4e48abc9771a8a9de0ad is 50, key is test_row_0/C:col10/1733815048088/Put/seqid=0 2024-12-10T07:17:29,434 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:29,443 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210b6ce48cff3714b89babb8d8e1822dee2_508b45081a20fbcb4fc7da445cb3ab9f store=[table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:29,445 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210b6ce48cff3714b89babb8d8e1822dee2_508b45081a20fbcb4fc7da445cb3ab9f, store=[table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:29,445 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210b6ce48cff3714b89babb8d8e1822dee2_508b45081a20fbcb4fc7da445cb3ab9f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:29,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742362_1538 (size=12493) 2024-12-10T07:17:29,452 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/db895434e3ee4e48abc9771a8a9de0ad as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/db895434e3ee4e48abc9771a8a9de0ad 2024-12-10T07:17:29,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121014888ebf245d4fc48cb429822f49025b_508b45081a20fbcb4fc7da445cb3ab9f is 50, key is test_row_0/A:col10/1733815048783/Put/seqid=0 2024-12-10T07:17:29,464 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 508b45081a20fbcb4fc7da445cb3ab9f/C of 508b45081a20fbcb4fc7da445cb3ab9f into db895434e3ee4e48abc9771a8a9de0ad(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:17:29,464 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:29,464 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., storeName=508b45081a20fbcb4fc7da445cb3ab9f/C, priority=13, startTime=1733815049410; duration=0sec 2024-12-10T07:17:29,464 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:17:29,464 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 508b45081a20fbcb4fc7da445cb3ab9f:C 2024-12-10T07:17:29,464 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:17:29,466 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:17:29,466 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 508b45081a20fbcb4fc7da445cb3ab9f/B is initiating minor compaction (all files) 2024-12-10T07:17:29,466 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 508b45081a20fbcb4fc7da445cb3ab9f/B in TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:29,466 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/9ec1b479822f46028921949a15c595c8, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/31461da1009e4abb917c8bf75f8570d2, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/11a62f303dd9472685c34a1e974ef804] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp, totalSize=35.7 K 2024-12-10T07:17:29,467 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ec1b479822f46028921949a15c595c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733815046136 2024-12-10T07:17:29,467 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31461da1009e4abb917c8bf75f8570d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733815047373 2024-12-10T07:17:29,467 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11a62f303dd9472685c34a1e974ef804, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733815048083 2024-12-10T07:17:29,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742363_1539 (size=4469) 2024-12-10T07:17:29,500 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 508b45081a20fbcb4fc7da445cb3ab9f#B#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:17:29,500 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/e2968c5dff9b4a4b98b7374908b6f4f7 is 50, key is test_row_0/B:col10/1733815048088/Put/seqid=0 2024-12-10T07:17:29,501 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 508b45081a20fbcb4fc7da445cb3ab9f#A#compaction#444 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T07:17:29,502 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/b02f033c40314e1f8d80b416edce6b08 is 175, key is test_row_0/A:col10/1733815048088/Put/seqid=0 2024-12-10T07:17:29,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742364_1540 (size=14794) 2024-12-10T07:17:29,519 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:17:29,524 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121014888ebf245d4fc48cb429822f49025b_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121014888ebf245d4fc48cb429822f49025b_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:29,525 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/4c1df781ddc74ff3bfc063ba415682be, store: [table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:29,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/4c1df781ddc74ff3bfc063ba415682be is 175, key is test_row_0/A:col10/1733815048783/Put/seqid=0 2024-12-10T07:17:29,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-12-10T07:17:29,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:29,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:29,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:29,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:29,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:29,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815109548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:29,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:29,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:29,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:29,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742366_1542 (size=31447) 2024-12-10T07:17:29,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742365_1541 (size=12493) 2024-12-10T07:17:29,570 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/b02f033c40314e1f8d80b416edce6b08 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/b02f033c40314e1f8d80b416edce6b08 2024-12-10T07:17:29,584 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/e2968c5dff9b4a4b98b7374908b6f4f7 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/e2968c5dff9b4a4b98b7374908b6f4f7 2024-12-10T07:17:29,585 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 508b45081a20fbcb4fc7da445cb3ab9f/A of 508b45081a20fbcb4fc7da445cb3ab9f into b02f033c40314e1f8d80b416edce6b08(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:17:29,585 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:29,585 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., storeName=508b45081a20fbcb4fc7da445cb3ab9f/A, priority=13, startTime=1733815049410; duration=0sec 2024-12-10T07:17:29,585 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:17:29,585 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 508b45081a20fbcb4fc7da445cb3ab9f:A 2024-12-10T07:17:29,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742367_1543 (size=39749) 2024-12-10T07:17:29,605 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 508b45081a20fbcb4fc7da445cb3ab9f/B of 508b45081a20fbcb4fc7da445cb3ab9f into e2968c5dff9b4a4b98b7374908b6f4f7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:17:29,605 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:29,605 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., storeName=508b45081a20fbcb4fc7da445cb3ab9f/B, priority=13, startTime=1733815049410; duration=0sec 2024-12-10T07:17:29,605 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:17:29,605 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 508b45081a20fbcb4fc7da445cb3ab9f:B 2024-12-10T07:17:29,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:29,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815109665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:29,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-12-10T07:17:29,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:29,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:29,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:29,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:29,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:29,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-12-10T07:17:29,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:29,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:29,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:29,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:29,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:29,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:29,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:29,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815109875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:29,992 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/4c1df781ddc74ff3bfc063ba415682be 2024-12-10T07:17:30,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/d063060cb6d4408cb3684e85e5e4aa83 is 50, key is test_row_0/B:col10/1733815048783/Put/seqid=0 2024-12-10T07:17:30,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-12-10T07:17:30,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:30,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:30,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:30,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:30,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:30,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742368_1544 (size=12151) 2024-12-10T07:17:30,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/d063060cb6d4408cb3684e85e5e4aa83 2024-12-10T07:17:30,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/a60bf8b4162547098f15081f30498f31 is 50, key is test_row_0/C:col10/1733815048783/Put/seqid=0 2024-12-10T07:17:30,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742369_1545 (size=12151) 2024-12-10T07:17:30,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/a60bf8b4162547098f15081f30498f31 2024-12-10T07:17:30,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/4c1df781ddc74ff3bfc063ba415682be as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/4c1df781ddc74ff3bfc063ba415682be 2024-12-10T07:17:30,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/4c1df781ddc74ff3bfc063ba415682be, entries=200, sequenceid=172, filesize=38.8 K 2024-12-10T07:17:30,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/d063060cb6d4408cb3684e85e5e4aa83 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/d063060cb6d4408cb3684e85e5e4aa83 2024-12-10T07:17:30,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/d063060cb6d4408cb3684e85e5e4aa83, entries=150, sequenceid=172, filesize=11.9 K 2024-12-10T07:17:30,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/a60bf8b4162547098f15081f30498f31 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/a60bf8b4162547098f15081f30498f31 2024-12-10T07:17:30,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/a60bf8b4162547098f15081f30498f31, entries=150, sequenceid=172, filesize=11.9 K 2024-12-10T07:17:30,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 508b45081a20fbcb4fc7da445cb3ab9f in 658ms, sequenceid=172, compaction requested=false 2024-12-10T07:17:30,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:30,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-12-10T07:17:30,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:30,164 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2902): Flushing 508b45081a20fbcb4fc7da445cb3ab9f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-10T07:17:30,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=A 2024-12-10T07:17:30,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:30,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=B 2024-12-10T07:17:30,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:30,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=C 2024-12-10T07:17:30,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:30,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a0ae5fae46e6496788d9be97f32b821d_508b45081a20fbcb4fc7da445cb3ab9f is 50, key is test_row_0/A:col10/1733815049516/Put/seqid=0 2024-12-10T07:17:30,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:30,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:30,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742370_1546 (size=12304) 2024-12-10T07:17:30,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:17:30,201 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210a0ae5fae46e6496788d9be97f32b821d_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a0ae5fae46e6496788d9be97f32b821d_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:30,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/105ae55b638f4a0d85bf7a843f7067c1, store: [table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:30,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/105ae55b638f4a0d85bf7a843f7067c1 is 175, key is test_row_0/A:col10/1733815049516/Put/seqid=0 2024-12-10T07:17:30,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742371_1547 (size=31105) 2024-12-10T07:17:30,235 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=191, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/105ae55b638f4a0d85bf7a843f7067c1 2024-12-10T07:17:30,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/fcaf97d3d1b346a08ac19bc01be66bf5 is 50, key is test_row_0/B:col10/1733815049516/Put/seqid=0 2024-12-10T07:17:30,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742372_1548 (size=12151) 2024-12-10T07:17:30,269 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/fcaf97d3d1b346a08ac19bc01be66bf5 2024-12-10T07:17:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-10T07:17:30,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815110282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:30,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:30,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:30,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:30,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/77ae2fd235624f40b71d9df31de82acf is 50, key is test_row_0/C:col10/1733815049516/Put/seqid=0 2024-12-10T07:17:30,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742373_1549 (size=12151) 2024-12-10T07:17:30,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:30,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815110396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:30,400 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:30,400 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:30,400 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:30,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:30,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56978 deadline: 1733815110456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:30,460 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 508b45081a20fbcb4fc7da445cb3ab9f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:17:30,461 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:30,461 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:30,461 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:30,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:30,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56968 deadline: 1733815110465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:30,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:30,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56902 deadline: 1733815110465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:30,468 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_0' in 508b45081a20fbcb4fc7da445cb3ab9f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:17:30,468 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:30,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:30,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:30,468 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_0' in 508b45081a20fbcb4fc7da445cb3ab9f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:17:30,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:30,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:30,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:30,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56910 deadline: 1733815110480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:30,481 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_2' in 508b45081a20fbcb4fc7da445cb3ab9f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:17:30,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:30,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:30,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:30,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815110605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:30,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:30,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:30,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:30,706 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/77ae2fd235624f40b71d9df31de82acf 2024-12-10T07:17:30,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/105ae55b638f4a0d85bf7a843f7067c1 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/105ae55b638f4a0d85bf7a843f7067c1 2024-12-10T07:17:30,715 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/105ae55b638f4a0d85bf7a843f7067c1, entries=150, sequenceid=191, filesize=30.4 K 2024-12-10T07:17:30,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/fcaf97d3d1b346a08ac19bc01be66bf5 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/fcaf97d3d1b346a08ac19bc01be66bf5 2024-12-10T07:17:30,719 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/fcaf97d3d1b346a08ac19bc01be66bf5, entries=150, sequenceid=191, filesize=11.9 K 2024-12-10T07:17:30,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/77ae2fd235624f40b71d9df31de82acf as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/77ae2fd235624f40b71d9df31de82acf 2024-12-10T07:17:30,725 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/77ae2fd235624f40b71d9df31de82acf, entries=150, sequenceid=191, filesize=11.9 K 2024-12-10T07:17:30,726 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 508b45081a20fbcb4fc7da445cb3ab9f in 562ms, sequenceid=191, compaction requested=true 2024-12-10T07:17:30,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2603): Flush status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:30,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:30,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=137 2024-12-10T07:17:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=137 2024-12-10T07:17:30,730 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-12-10T07:17:30,730 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5980 sec 2024-12-10T07:17:30,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees in 2.6060 sec 2024-12-10T07:17:30,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 508b45081a20fbcb4fc7da445cb3ab9f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-10T07:17:30,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=A 2024-12-10T07:17:30,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:30,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=B 2024-12-10T07:17:30,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:30,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=C 2024-12-10T07:17:30,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:30,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:30,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121037e3c97fa8a8409ca488f5951b1f6573_508b45081a20fbcb4fc7da445cb3ab9f is 50, key is test_row_0/A:col10/1733815050272/Put/seqid=0 2024-12-10T07:17:30,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742374_1550 (size=14794) 2024-12-10T07:17:30,938 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:17:30,941 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121037e3c97fa8a8409ca488f5951b1f6573_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121037e3c97fa8a8409ca488f5951b1f6573_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:30,942 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/4882d3874a5749908fc89e9d4e341d05, store: [table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:30,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/4882d3874a5749908fc89e9d4e341d05 is 175, key is test_row_0/A:col10/1733815050272/Put/seqid=0 2024-12-10T07:17:30,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742375_1551 (size=39749) 2024-12-10T07:17:30,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815110990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:30,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:30,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:30,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:31,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815111106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:31,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:31,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:31,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:31,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:31,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815111315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:31,319 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:31,319 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:31,319 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:31,355 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/4882d3874a5749908fc89e9d4e341d05 2024-12-10T07:17:31,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/db0994a1073d4a11bf6ae87ed39b2c1e is 50, key is test_row_0/B:col10/1733815050272/Put/seqid=0 2024-12-10T07:17:31,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742376_1552 (size=12151) 2024-12-10T07:17:31,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815111625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:31,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:31,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:31,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:32,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:32,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815112200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:32,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:32,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:32,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:32,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-10T07:17:32,294 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:17:32,295 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:17:33,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:33,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815113214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:33,217 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 508b45081a20fbcb4fc7da445cb3ab9f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:17:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:33,711 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T07:17:35,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:35,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815115224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:35,227 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 508b45081a20fbcb4fc7da445cb3ab9f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:17:35,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:35,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:35,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:37,815 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1803234487_22 at /127.0.0.1:60154 [Receiving block BP-1692125974-172.17.0.3-1733814870213:blk_1073741830_1006] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 5520ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/cluster_ac824e32-1cbb-51f2-a594-edd33703f97a/data/data2/, blockId=1073741830, seqno=1769 2024-12-10T07:17:37,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/db0994a1073d4a11bf6ae87ed39b2c1e 2024-12-10T07:17:37,815 INFO [AsyncFSWAL-0-hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData-prefix:b1396b8d2d12,36259,1733814874775 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5520 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40613,DS-71c3e2fe-e06c-4f2c-995a-a2f36a2ad6fa,DISK]] 2024-12-10T07:17:37,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees 2024-12-10T07:17:37,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-10T07:17:37,819 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:17:37,820 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:17:37,820 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:17:37,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/5fe78d53af0b4568ab3f895bc09816c3 is 50, key is test_row_0/C:col10/1733815050272/Put/seqid=0 2024-12-10T07:17:37,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742377_1553 (size=12151) 2024-12-10T07:17:37,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-10T07:17:37,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-10T07:17:37,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:37,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:37,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:37,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:37,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:38,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-10T07:17:38,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:38,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-10T07:17:38,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:38,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:38,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:38,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:38,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T07:17:38,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/5fe78d53af0b4568ab3f895bc09816c3 2024-12-10T07:17:38,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/4882d3874a5749908fc89e9d4e341d05 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/4882d3874a5749908fc89e9d4e341d05 2024-12-10T07:17:38,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/4882d3874a5749908fc89e9d4e341d05, entries=200, sequenceid=208, filesize=38.8 K 2024-12-10T07:17:38,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/db0994a1073d4a11bf6ae87ed39b2c1e as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/db0994a1073d4a11bf6ae87ed39b2c1e 2024-12-10T07:17:38,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/db0994a1073d4a11bf6ae87ed39b2c1e, entries=150, sequenceid=208, filesize=11.9 K 2024-12-10T07:17:38,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/5fe78d53af0b4568ab3f895bc09816c3 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/5fe78d53af0b4568ab3f895bc09816c3 2024-12-10T07:17:38,270 DEBUG [regionserver/b1396b8d2d12:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2024-12-10T07:17:38,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/5fe78d53af0b4568ab3f895bc09816c3, entries=150, sequenceid=208, filesize=11.9 K 2024-12-10T07:17:38,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 508b45081a20fbcb4fc7da445cb3ab9f in 7354ms, sequenceid=208, compaction requested=true 2024-12-10T07:17:38,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:38,277 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:17:38,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 508b45081a20fbcb4fc7da445cb3ab9f:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:17:38,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:17:38,278 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:17:38,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 508b45081a20fbcb4fc7da445cb3ab9f:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:17:38,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:17:38,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 508b45081a20fbcb4fc7da445cb3ab9f:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:17:38,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:17:38,279 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142050 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:17:38,279 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 508b45081a20fbcb4fc7da445cb3ab9f/A is initiating minor compaction (all files) 2024-12-10T07:17:38,279 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 508b45081a20fbcb4fc7da445cb3ab9f/A in TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:38,279 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/b02f033c40314e1f8d80b416edce6b08, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/4c1df781ddc74ff3bfc063ba415682be, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/105ae55b638f4a0d85bf7a843f7067c1, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/4882d3874a5749908fc89e9d4e341d05] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp, totalSize=138.7 K 2024-12-10T07:17:38,279 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:38,279 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. files: [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/b02f033c40314e1f8d80b416edce6b08, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/4c1df781ddc74ff3bfc063ba415682be, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/105ae55b638f4a0d85bf7a843f7067c1, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/4882d3874a5749908fc89e9d4e341d05] 2024-12-10T07:17:38,279 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:17:38,279 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting b02f033c40314e1f8d80b416edce6b08, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733815048083 2024-12-10T07:17:38,279 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 508b45081a20fbcb4fc7da445cb3ab9f/B is initiating minor compaction (all files) 2024-12-10T07:17:38,280 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 508b45081a20fbcb4fc7da445cb3ab9f/B in TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:38,280 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/e2968c5dff9b4a4b98b7374908b6f4f7, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/d063060cb6d4408cb3684e85e5e4aa83, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/fcaf97d3d1b346a08ac19bc01be66bf5, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/db0994a1073d4a11bf6ae87ed39b2c1e] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp, totalSize=47.8 K 2024-12-10T07:17:38,280 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c1df781ddc74ff3bfc063ba415682be, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733815048767 2024-12-10T07:17:38,280 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 105ae55b638f4a0d85bf7a843f7067c1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733815049516 2024-12-10T07:17:38,280 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting e2968c5dff9b4a4b98b7374908b6f4f7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733815048083 2024-12-10T07:17:38,280 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting d063060cb6d4408cb3684e85e5e4aa83, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733815048767 2024-12-10T07:17:38,280 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4882d3874a5749908fc89e9d4e341d05, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733815050266 2024-12-10T07:17:38,281 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting fcaf97d3d1b346a08ac19bc01be66bf5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733815049516 2024-12-10T07:17:38,281 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting db0994a1073d4a11bf6ae87ed39b2c1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733815050272 2024-12-10T07:17:38,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-12-10T07:17:38,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:38,293 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2902): Flushing 508b45081a20fbcb4fc7da445cb3ab9f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-10T07:17:38,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=A 2024-12-10T07:17:38,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:38,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=B 2024-12-10T07:17:38,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:38,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=C 2024-12-10T07:17:38,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:38,299 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 508b45081a20fbcb4fc7da445cb3ab9f#B#compaction#455 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T07:17:38,299 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/3288bc810c9a4b9b80eb19bfdb249919 is 50, key is test_row_0/B:col10/1733815050272/Put/seqid=0 2024-12-10T07:17:38,312 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:38,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121052cb84c551d445af9bea523814c606ab_508b45081a20fbcb4fc7da445cb3ab9f is 50, key is test_row_0/A:col10/1733815050964/Put/seqid=0 2024-12-10T07:17:38,338 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241210686f02bdd4a54df78a9e55e7fe02f9fc_508b45081a20fbcb4fc7da445cb3ab9f store=[table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:38,340 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241210686f02bdd4a54df78a9e55e7fe02f9fc_508b45081a20fbcb4fc7da445cb3ab9f, store=[table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:38,341 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241210686f02bdd4a54df78a9e55e7fe02f9fc_508b45081a20fbcb4fc7da445cb3ab9f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:38,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742379_1555 (size=12304) 2024-12-10T07:17:38,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742378_1554 (size=12629) 2024-12-10T07:17:38,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742380_1556 (size=4469) 2024-12-10T07:17:38,367 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 508b45081a20fbcb4fc7da445cb3ab9f#A#compaction#456 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T07:17:38,367 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/5073a8c714b24617b55d1f78424e377a is 175, key is test_row_0/A:col10/1733815050272/Put/seqid=0 2024-12-10T07:17:38,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742381_1557 (size=31583) 2024-12-10T07:17:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-10T07:17:38,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:17:38,753 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121052cb84c551d445af9bea523814c606ab_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121052cb84c551d445af9bea523814c606ab_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:38,754 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/3288bc810c9a4b9b80eb19bfdb249919 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/3288bc810c9a4b9b80eb19bfdb249919 2024-12-10T07:17:38,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/01fd3f9fc9344034b04a41c7c295f65c, store: [table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:38,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/01fd3f9fc9344034b04a41c7c295f65c is 175, key is test_row_0/A:col10/1733815050964/Put/seqid=0 2024-12-10T07:17:38,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742382_1558 (size=31105) 2024-12-10T07:17:38,760 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 508b45081a20fbcb4fc7da445cb3ab9f/B of 508b45081a20fbcb4fc7da445cb3ab9f into 3288bc810c9a4b9b80eb19bfdb249919(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:17:38,760 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:38,760 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., storeName=508b45081a20fbcb4fc7da445cb3ab9f/B, priority=12, startTime=1733815058278; duration=0sec 2024-12-10T07:17:38,760 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:17:38,760 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 508b45081a20fbcb4fc7da445cb3ab9f:B 2024-12-10T07:17:38,760 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T07:17:38,761 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-10T07:17:38,761 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 508b45081a20fbcb4fc7da445cb3ab9f/C is initiating minor compaction (all files) 2024-12-10T07:17:38,761 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 508b45081a20fbcb4fc7da445cb3ab9f/C in TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:38,761 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/db895434e3ee4e48abc9771a8a9de0ad, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/a60bf8b4162547098f15081f30498f31, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/77ae2fd235624f40b71d9df31de82acf, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/5fe78d53af0b4568ab3f895bc09816c3] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp, totalSize=47.8 K 2024-12-10T07:17:38,762 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting db895434e3ee4e48abc9771a8a9de0ad, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733815048083 2024-12-10T07:17:38,762 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting a60bf8b4162547098f15081f30498f31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733815048767 2024-12-10T07:17:38,762 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 77ae2fd235624f40b71d9df31de82acf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1733815049516 2024-12-10T07:17:38,763 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fe78d53af0b4568ab3f895bc09816c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733815050272 2024-12-10T07:17:38,770 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 508b45081a20fbcb4fc7da445cb3ab9f#C#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T07:17:38,770 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/b4bd3c8ace024c4c8f2f4b4b737a5a98 is 50, key is test_row_0/C:col10/1733815050272/Put/seqid=0 2024-12-10T07:17:38,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742383_1559 (size=12629) 2024-12-10T07:17:38,794 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/5073a8c714b24617b55d1f78424e377a as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/5073a8c714b24617b55d1f78424e377a 2024-12-10T07:17:38,797 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 508b45081a20fbcb4fc7da445cb3ab9f/A of 508b45081a20fbcb4fc7da445cb3ab9f into 5073a8c714b24617b55d1f78424e377a(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:17:38,797 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:38,797 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., storeName=508b45081a20fbcb4fc7da445cb3ab9f/A, priority=12, startTime=1733815058277; duration=0sec 2024-12-10T07:17:38,797 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:17:38,797 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 508b45081a20fbcb4fc7da445cb3ab9f:A 2024-12-10T07:17:38,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-10T07:17:39,160 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=223, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/01fd3f9fc9344034b04a41c7c295f65c 2024-12-10T07:17:39,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/208960e0719a4916abe585738784484f is 50, key is test_row_0/B:col10/1733815050964/Put/seqid=0 2024-12-10T07:17:39,211 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/b4bd3c8ace024c4c8f2f4b4b737a5a98 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/b4bd3c8ace024c4c8f2f4b4b737a5a98 2024-12-10T07:17:39,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742384_1560 (size=12151) 2024-12-10T07:17:39,214 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/208960e0719a4916abe585738784484f 2024-12-10T07:17:39,223 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 508b45081a20fbcb4fc7da445cb3ab9f/C of 508b45081a20fbcb4fc7da445cb3ab9f into b4bd3c8ace024c4c8f2f4b4b737a5a98(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:17:39,223 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:39,223 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., storeName=508b45081a20fbcb4fc7da445cb3ab9f/C, priority=12, startTime=1733815058278; duration=0sec 2024-12-10T07:17:39,223 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:17:39,223 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 508b45081a20fbcb4fc7da445cb3ab9f:C 2024-12-10T07:17:39,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/5e2534fa17f4422e909f4b98700f930b is 50, key is test_row_0/C:col10/1733815050964/Put/seqid=0 2024-12-10T07:17:39,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:39,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:39,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742385_1561 (size=12151) 2024-12-10T07:17:39,271 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/5e2534fa17f4422e909f4b98700f930b 2024-12-10T07:17:39,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/01fd3f9fc9344034b04a41c7c295f65c as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/01fd3f9fc9344034b04a41c7c295f65c 2024-12-10T07:17:39,285 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/01fd3f9fc9344034b04a41c7c295f65c, entries=150, sequenceid=223, filesize=30.4 K 2024-12-10T07:17:39,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/208960e0719a4916abe585738784484f as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/208960e0719a4916abe585738784484f 2024-12-10T07:17:39,297 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/208960e0719a4916abe585738784484f, entries=150, sequenceid=223, filesize=11.9 K 2024-12-10T07:17:39,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/5e2534fa17f4422e909f4b98700f930b as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/5e2534fa17f4422e909f4b98700f930b 2024-12-10T07:17:39,312 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/5e2534fa17f4422e909f4b98700f930b, entries=150, sequenceid=223, filesize=11.9 K 2024-12-10T07:17:39,313 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=67.09 KB/68700 for 508b45081a20fbcb4fc7da445cb3ab9f in 1020ms, sequenceid=223, compaction requested=false 2024-12-10T07:17:39,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2603): Flush status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:39,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:39,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=139 2024-12-10T07:17:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=139 2024-12-10T07:17:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:39,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 508b45081a20fbcb4fc7da445cb3ab9f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-10T07:17:39,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=A 2024-12-10T07:17:39,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:39,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=B 2024-12-10T07:17:39,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:39,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=C 2024-12-10T07:17:39,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:39,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-10T07:17:39,318 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4950 sec 2024-12-10T07:17:39,321 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees in 7.0230 sec 2024-12-10T07:17:39,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412106e61dcfc584245b1864aac511cbef050_508b45081a20fbcb4fc7da445cb3ab9f is 50, key is test_row_0/A:col10/1733815059309/Put/seqid=0 2024-12-10T07:17:39,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742386_1562 (size=12304) 2024-12-10T07:17:39,393 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:17:39,432 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412106e61dcfc584245b1864aac511cbef050_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412106e61dcfc584245b1864aac511cbef050_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:39,437 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/d064f5d5c9c2406ea1b0b3cedd7fc224, store: [table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:39,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/d064f5d5c9c2406ea1b0b3cedd7fc224 is 175, key is test_row_0/A:col10/1733815059309/Put/seqid=0 2024-12-10T07:17:39,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742387_1563 (size=31105) 2024-12-10T07:17:39,497 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/d064f5d5c9c2406ea1b0b3cedd7fc224 2024-12-10T07:17:39,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815119533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:39,535 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:39,535 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:39,535 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:39,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/0841642bd15f4f54b0397c0f0ca39deb is 50, key is test_row_0/B:col10/1733815059309/Put/seqid=0 2024-12-10T07:17:39,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742388_1564 (size=12151) 2024-12-10T07:17:39,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/0841642bd15f4f54b0397c0f0ca39deb 2024-12-10T07:17:39,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/5547cd3a2960451786b3cddb93bb3fc4 is 50, key is test_row_0/C:col10/1733815059309/Put/seqid=0 2024-12-10T07:17:39,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742389_1565 (size=12151) 2024-12-10T07:17:39,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:39,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815119646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:39,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:39,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:39,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:39,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815119855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-10T07:17:39,966 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:17:39,967 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-10T07:17:40,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815120166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:40,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:40,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56978 deadline: 1733815120475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:40,477 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_1' in 508b45081a20fbcb4fc7da445cb3ab9f of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:17:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:40,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:40,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56902 deadline: 1733815120485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:40,486 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_0' in 508b45081a20fbcb4fc7da445cb3ab9f of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:17:40,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:40,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:40,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:40,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:40,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56968 deadline: 1733815120575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:40,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:40,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56910 deadline: 1733815120575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:40,577 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_0' in 508b45081a20fbcb4fc7da445cb3ab9f of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18325 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:17:40,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:40,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:40,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:40,578 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_2' in 508b45081a20fbcb4fc7da445cb3ab9f of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18320 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:17:40,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:40,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:40,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:40,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815120675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:41,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:41,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815121684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:41,687 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_0' in 508b45081a20fbcb4fc7da445cb3ab9f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:17:41,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:41,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:41,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:41,851 INFO [Thread-2255 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2255. 2024-12-10T07:17:41,851 DEBUG [Thread-2255 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:17:41,851 DEBUG [Thread-2255 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:41,851 DEBUG [Thread-2255 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:41,851 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:17:41,851 INFO [Thread-2261 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2261. 2024-12-10T07:17:41,851 DEBUG [Thread-2261 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:17:41,851 DEBUG [Thread-2261 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:41,851 DEBUG [Thread-2261 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:41,851 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:17:41,852 INFO [Thread-2263 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2263. 2024-12-10T07:17:41,852 DEBUG [Thread-2263 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:17:41,852 DEBUG [Thread-2263 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:41,853 DEBUG [Thread-2263 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:41,853 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:17:41,853 INFO [Thread-2257 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2257. 2024-12-10T07:17:41,853 DEBUG [Thread-2257 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:17:41,853 DEBUG [Thread-2257 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:41,853 DEBUG [Thread-2257 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:41,853 INFO [Thread-2259 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2259. 2024-12-10T07:17:41,854 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:17:41,854 DEBUG [Thread-2259 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:17:41,854 DEBUG [Thread-2259 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:41,854 DEBUG [Thread-2259 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:41,854 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:17:43,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-10T07:17:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56966 deadline: 1733815123706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 2024-12-10T07:17:43,708 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to b1396b8d2d12,35871,1733814875835 for 'test_row_0' in 508b45081a20fbcb4fc7da445cb3ab9f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-10T07:17:43,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 , the old value is region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:43,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-10T07:17:43,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., hostname=b1396b8d2d12,35871,1733814875835, seqNum=5 because the exception is null or not the one we care about 2024-12-10T07:17:46,335 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-1803234487_22 at /127.0.0.1:60154 [Receiving block BP-1692125974-172.17.0.3-1733814870213:blk_1073741830_1006] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 6367ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/cluster_ac824e32-1cbb-51f2-a594-edd33703f97a/data/data2/, blockId=1073741830, seqno=1803 2024-12-10T07:17:46,336 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] monitor.StreamSlowMonitor(154): Slow datanode: DatanodeInfoWithStorage[127.0.0.1:40613,DS-71c3e2fe-e06c-4f2c-995a-a2f36a2ad6fa,DISK], data length=853, duration=6369ms, unfinishedReplicas=0, lastAckTimestamp=-1, monitor name: defaultMonitorName 2024-12-10T07:17:46,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/5547cd3a2960451786b3cddb93bb3fc4 2024-12-10T07:17:46,339 INFO [AsyncFSWAL-0-hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData-prefix:b1396b8d2d12,36259,1733814874775 {}] wal.AbstractFSWAL(1368): Slow sync cost: 6371 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40613,DS-71c3e2fe-e06c-4f2c-995a-a2f36a2ad6fa,DISK]] 2024-12-10T07:17:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-12-10T07:17:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-10T07:17:46,341 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-10T07:17:46,342 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-10T07:17:46,342 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-10T07:17:46,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/d064f5d5c9c2406ea1b0b3cedd7fc224 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/d064f5d5c9c2406ea1b0b3cedd7fc224 2024-12-10T07:17:46,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/d064f5d5c9c2406ea1b0b3cedd7fc224, entries=150, sequenceid=240, filesize=30.4 K 2024-12-10T07:17:46,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/0841642bd15f4f54b0397c0f0ca39deb as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/0841642bd15f4f54b0397c0f0ca39deb 2024-12-10T07:17:46,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/0841642bd15f4f54b0397c0f0ca39deb, entries=150, sequenceid=240, filesize=11.9 K 2024-12-10T07:17:46,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/5547cd3a2960451786b3cddb93bb3fc4 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/5547cd3a2960451786b3cddb93bb3fc4 2024-12-10T07:17:46,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/5547cd3a2960451786b3cddb93bb3fc4, entries=150, sequenceid=240, filesize=11.9 K 2024-12-10T07:17:46,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 508b45081a20fbcb4fc7da445cb3ab9f in 7038ms, sequenceid=240, compaction requested=true 2024-12-10T07:17:46,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:46,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 508b45081a20fbcb4fc7da445cb3ab9f:A, priority=-2147483648, current under compaction store size is 1 2024-12-10T07:17:46,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:17:46,355 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:17:46,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 508b45081a20fbcb4fc7da445cb3ab9f:B, priority=-2147483648, current under compaction store size is 2 2024-12-10T07:17:46,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:17:46,355 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:17:46,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 508b45081a20fbcb4fc7da445cb3ab9f:C, priority=-2147483648, current under compaction store size is 3 2024-12-10T07:17:46,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:17:46,355 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:17:46,355 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93793 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:17:46,355 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 508b45081a20fbcb4fc7da445cb3ab9f/B is initiating minor compaction (all files) 2024-12-10T07:17:46,356 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1541): 508b45081a20fbcb4fc7da445cb3ab9f/A is initiating minor compaction (all files) 2024-12-10T07:17:46,356 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 508b45081a20fbcb4fc7da445cb3ab9f/A in TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:46,356 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 508b45081a20fbcb4fc7da445cb3ab9f/B in TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:46,356 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/3288bc810c9a4b9b80eb19bfdb249919, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/208960e0719a4916abe585738784484f, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/0841642bd15f4f54b0397c0f0ca39deb] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp, totalSize=36.1 K 2024-12-10T07:17:46,356 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/5073a8c714b24617b55d1f78424e377a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/01fd3f9fc9344034b04a41c7c295f65c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/d064f5d5c9c2406ea1b0b3cedd7fc224] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp, totalSize=91.6 K 2024-12-10T07:17:46,356 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:46,356 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. files: [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/5073a8c714b24617b55d1f78424e377a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/01fd3f9fc9344034b04a41c7c295f65c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/d064f5d5c9c2406ea1b0b3cedd7fc224] 2024-12-10T07:17:46,356 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 3288bc810c9a4b9b80eb19bfdb249919, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733815050272 2024-12-10T07:17:46,356 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5073a8c714b24617b55d1f78424e377a, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733815050272 2024-12-10T07:17:46,356 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 208960e0719a4916abe585738784484f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733815050964 2024-12-10T07:17:46,356 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01fd3f9fc9344034b04a41c7c295f65c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733815050964 2024-12-10T07:17:46,356 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 0841642bd15f4f54b0397c0f0ca39deb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733815059260 2024-12-10T07:17:46,356 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] compactions.Compactor(225): Compacting d064f5d5c9c2406ea1b0b3cedd7fc224, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733815059260 2024-12-10T07:17:46,360 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:46,361 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 508b45081a20fbcb4fc7da445cb3ab9f#B#compaction#464 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-10T07:17:46,362 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/618cbeaaa08d4344bd3c65cfecf50048 is 50, key is test_row_0/B:col10/1733815059309/Put/seqid=0 2024-12-10T07:17:46,362 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121001ba215ee50f46b6a4d9991ddacba49c_508b45081a20fbcb4fc7da445cb3ab9f store=[table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:46,363 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121001ba215ee50f46b6a4d9991ddacba49c_508b45081a20fbcb4fc7da445cb3ab9f, store=[table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:46,363 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121001ba215ee50f46b6a4d9991ddacba49c_508b45081a20fbcb4fc7da445cb3ab9f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-10T07:17:46,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-12-10T07:17:46,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:46,496 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2902): Flushing 508b45081a20fbcb4fc7da445cb3ab9f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-10T07:17:46,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=A 2024-12-10T07:17:46,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:46,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=B 2024-12-10T07:17:46,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:46,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=C 2024-12-10T07:17:46,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:17:46,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121082cf365692d6467eb59d28ff4270b63b_508b45081a20fbcb4fc7da445cb3ab9f is 50, key is test_row_0/A:col10/1733815059524/Put/seqid=0 2024-12-10T07:17:46,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-10T07:17:46,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-10T07:17:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-10T07:17:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35871 {}] regionserver.HRegion(8855): Flush requested on 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:47,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. as already flushing 2024-12-10T07:17:47,757 INFO [Thread-2250 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2250. 2024-12-10T07:17:47,757 DEBUG [Thread-2250 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:17:47,757 DEBUG [Thread-2250 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:47,757 DEBUG [Thread-2250 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:47,757 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:17:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-10T07:17:50,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-10T07:17:50,738 INFO [AsyncFSWAL-0-hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8-prefix:b1396b8d2d12,35871,1733814875835 {}] wal.AbstractFSWAL(1368): Slow sync cost: 170 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40613,DS-71c3e2fe-e06c-4f2c-995a-a2f36a2ad6fa,DISK]] 2024-12-10T07:17:50,738 INFO [AsyncFSWAL-0-hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8-prefix:b1396b8d2d12,35871,1733814875835 {}] wal.AbstractFSWAL(1368): Slow sync cost: 161 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40613,DS-71c3e2fe-e06c-4f2c-995a-a2f36a2ad6fa,DISK]] 2024-12-10T07:17:50,739 INFO [Thread-2248 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2248. 2024-12-10T07:17:50,739 DEBUG [Thread-2248 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:17:50,740 DEBUG [Thread-2248 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:50,740 DEBUG [Thread-2248 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:50,740 INFO [Thread-2244 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2244. 2024-12-10T07:17:50,740 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:17:50,740 DEBUG [Thread-2244 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:17:50,740 DEBUG [Thread-2244 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:50,741 DEBUG [Thread-2244 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:50,741 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:17:50,743 INFO [Thread-2252 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2252. 2024-12-10T07:17:50,743 INFO [Thread-2246 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2246. 2024-12-10T07:17:50,743 DEBUG [Thread-2246 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:17:50,743 DEBUG [Thread-2252 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-10T07:17:50,743 DEBUG [Thread-2246 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:50,743 DEBUG [Thread-2252 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:50,743 DEBUG [Thread-2246 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:50,743 DEBUG [Thread-2252 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:17:50,743 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:17:50,743 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:17:51,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742391_1567 (size=4469) 2024-12-10T07:17:51,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742390_1566 (size=12731) 2024-12-10T07:17:51,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742392_1568 (size=12354) 2024-12-10T07:17:52,178 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 508b45081a20fbcb4fc7da445cb3ab9f#A#compaction#465 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T07:17:52,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:17:52,178 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/f599dab02ffb444e9831c37eae53b0c7 is 175, key is test_row_0/A:col10/1733815059309/Put/seqid=0 2024-12-10T07:17:52,182 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/618cbeaaa08d4344bd3c65cfecf50048 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/618cbeaaa08d4344bd3c65cfecf50048 2024-12-10T07:17:52,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742393_1569 (size=31685) 2024-12-10T07:17:52,183 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121082cf365692d6467eb59d28ff4270b63b_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121082cf365692d6467eb59d28ff4270b63b_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:17:52,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/319cf69ae32640928f163cd14128255b, store: [table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:17:52,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/319cf69ae32640928f163cd14128255b is 175, key is test_row_0/A:col10/1733815059524/Put/seqid=0 2024-12-10T07:17:52,185 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 508b45081a20fbcb4fc7da445cb3ab9f/B of 508b45081a20fbcb4fc7da445cb3ab9f into 618cbeaaa08d4344bd3c65cfecf50048(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 5sec to execute. 2024-12-10T07:17:52,185 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:52,185 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., storeName=508b45081a20fbcb4fc7da445cb3ab9f/B, priority=13, startTime=1733815066355; duration=5sec 2024-12-10T07:17:52,185 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-10T07:17:52,185 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 508b45081a20fbcb4fc7da445cb3ab9f:B 2024-12-10T07:17:52,186 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-10T07:17:52,186 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-10T07:17:52,186 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1541): 508b45081a20fbcb4fc7da445cb3ab9f/C is initiating minor compaction (all files) 2024-12-10T07:17:52,186 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 508b45081a20fbcb4fc7da445cb3ab9f/C in TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:52,186 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/b4bd3c8ace024c4c8f2f4b4b737a5a98, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/5e2534fa17f4422e909f4b98700f930b, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/5547cd3a2960451786b3cddb93bb3fc4] into tmpdir=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp, totalSize=36.1 K 2024-12-10T07:17:52,187 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting b4bd3c8ace024c4c8f2f4b4b737a5a98, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733815050272 2024-12-10T07:17:52,187 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e2534fa17f4422e909f4b98700f930b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733815050964 2024-12-10T07:17:52,187 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] compactions.Compactor(225): Compacting 5547cd3a2960451786b3cddb93bb3fc4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733815059260 2024-12-10T07:17:52,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742394_1570 (size=31155) 2024-12-10T07:17:52,193 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 508b45081a20fbcb4fc7da445cb3ab9f#C#compaction#467 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-10T07:17:52,193 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/13877906c1334f2eb1098e31106d632b is 50, key is test_row_0/C:col10/1733815059309/Put/seqid=0 2024-12-10T07:17:52,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742395_1571 (size=12731) 2024-12-10T07:17:52,589 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/319cf69ae32640928f163cd14128255b 2024-12-10T07:17:52,591 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/f599dab02ffb444e9831c37eae53b0c7 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/f599dab02ffb444e9831c37eae53b0c7 2024-12-10T07:17:52,597 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 508b45081a20fbcb4fc7da445cb3ab9f/A of 508b45081a20fbcb4fc7da445cb3ab9f into f599dab02ffb444e9831c37eae53b0c7(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 6sec to execute. 2024-12-10T07:17:52,597 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:52,597 INFO [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., storeName=508b45081a20fbcb4fc7da445cb3ab9f/A, priority=13, startTime=1733815066355; duration=6sec 2024-12-10T07:17:52,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/1375feac46fd403da5b0fabf06993267 is 50, key is test_row_0/B:col10/1733815059524/Put/seqid=0 2024-12-10T07:17:52,597 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:17:52,597 DEBUG [RS:0;b1396b8d2d12:35871-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 508b45081a20fbcb4fc7da445cb3ab9f:A 2024-12-10T07:17:52,599 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/13877906c1334f2eb1098e31106d632b as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/13877906c1334f2eb1098e31106d632b 2024-12-10T07:17:52,602 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 508b45081a20fbcb4fc7da445cb3ab9f/C of 508b45081a20fbcb4fc7da445cb3ab9f into 13877906c1334f2eb1098e31106d632b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T07:17:52,602 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:52,602 INFO [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., storeName=508b45081a20fbcb4fc7da445cb3ab9f/C, priority=13, startTime=1733815066355; duration=0sec 2024-12-10T07:17:52,602 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-10T07:17:52,602 DEBUG [RS:0;b1396b8d2d12:35871-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 508b45081a20fbcb4fc7da445cb3ab9f:C 2024-12-10T07:17:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-10T07:17:55,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742396_1572 (size=12201) 2024-12-10T07:17:55,872 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/1375feac46fd403da5b0fabf06993267 2024-12-10T07:17:55,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/a0fb97fa886a4a63819995d70524c1b8 is 50, key is test_row_0/C:col10/1733815059524/Put/seqid=0 2024-12-10T07:17:55,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742397_1573 (size=12201) 2024-12-10T07:17:56,284 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/a0fb97fa886a4a63819995d70524c1b8 2024-12-10T07:17:56,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/319cf69ae32640928f163cd14128255b as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/319cf69ae32640928f163cd14128255b 2024-12-10T07:17:56,294 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/319cf69ae32640928f163cd14128255b, entries=150, sequenceid=258, filesize=30.4 K 2024-12-10T07:17:56,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/1375feac46fd403da5b0fabf06993267 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/1375feac46fd403da5b0fabf06993267 2024-12-10T07:17:56,297 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/1375feac46fd403da5b0fabf06993267, entries=150, sequenceid=258, filesize=11.9 K 2024-12-10T07:17:56,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/a0fb97fa886a4a63819995d70524c1b8 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/a0fb97fa886a4a63819995d70524c1b8 2024-12-10T07:17:56,300 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/a0fb97fa886a4a63819995d70524c1b8, entries=150, sequenceid=258, filesize=11.9 K 2024-12-10T07:17:56,301 INFO [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=33.54 KB/34350 for 508b45081a20fbcb4fc7da445cb3ab9f in 9805ms, sequenceid=258, compaction requested=false 2024-12-10T07:17:56,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2603): Flush status journal for 508b45081a20fbcb4fc7da445cb3ab9f: 2024-12-10T07:17:56,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:17:56,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b1396b8d2d12:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-12-10T07:17:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster(4169): Remote procedure done, pid=141 2024-12-10T07:17:56,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-12-10T07:17:56,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 9.9600 sec 2024-12-10T07:17:56,305 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 16.3370 sec 2024-12-10T07:18:03,711 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T07:18:04,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-10T07:18:04,615 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:18:04,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-10T07:18:04,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-12-10T07:18:04,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-12-10T07:18:04,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 6 2024-12-10T07:18:04,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 158 2024-12-10T07:18:04,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-12-10T07:18:04,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-10T07:18:04,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3553 2024-12-10T07:18:04,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3335 2024-12-10T07:18:04,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3363 2024-12-10T07:18:04,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3556 2024-12-10T07:18:04,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3362 2024-12-10T07:18:04,615 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-10T07:18:04,615 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T07:18:04,615 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T07:18:04,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:18:04,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:18:04,615 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:18:04,616 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-10T07:18:04,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-10T07:18:04,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-10T07:18:04,621 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733815084621"}]},"ts":"1733815084621"} 2024-12-10T07:18:04,623 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-10T07:18:04,624 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-10T07:18:04,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-10T07:18:04,627 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=508b45081a20fbcb4fc7da445cb3ab9f, UNASSIGN}] 2024-12-10T07:18:04,628 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=144, ppid=143, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=508b45081a20fbcb4fc7da445cb3ab9f, UNASSIGN 2024-12-10T07:18:04,629 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=144 updating hbase:meta row=508b45081a20fbcb4fc7da445cb3ab9f, regionState=CLOSING, regionLocation=b1396b8d2d12,35871,1733814875835 2024-12-10T07:18:04,632 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=508b45081a20fbcb4fc7da445cb3ab9f, UNASSIGN because future has completed 2024-12-10T07:18:04,632 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T07:18:04,632 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; CloseRegionProcedure 508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835}] 2024-12-10T07:18:04,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-10T07:18:04,789 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] handler.UnassignRegionHandler(122): Close 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:18:04,789 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T07:18:04,789 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegion(1722): Closing 508b45081a20fbcb4fc7da445cb3ab9f, disabling compactions & flushes 2024-12-10T07:18:04,789 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:18:04,789 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:18:04,789 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. after waiting 0 ms 2024-12-10T07:18:04,789 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:18:04,789 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegion(2902): Flushing 508b45081a20fbcb4fc7da445cb3ab9f 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-10T07:18:04,790 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=A 2024-12-10T07:18:04,790 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:18:04,790 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=B 2024-12-10T07:18:04,790 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:18:04,790 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 508b45081a20fbcb4fc7da445cb3ab9f, store=C 2024-12-10T07:18:04,790 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-10T07:18:04,797 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121011a499c439eb4999bedcbba2cbfe119f_508b45081a20fbcb4fc7da445cb3ab9f is 50, key is test_row_0/A:col10/1733815070739/Put/seqid=0 2024-12-10T07:18:04,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742398_1574 (size=12454) 2024-12-10T07:18:04,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-10T07:18:05,202 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T07:18:05,206 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121011a499c439eb4999bedcbba2cbfe119f_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121011a499c439eb4999bedcbba2cbfe119f_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:18:05,207 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/dd27f41ce71e4004a09589a2e1d3a78d, store: [table=TestAcidGuarantees family=A region=508b45081a20fbcb4fc7da445cb3ab9f] 2024-12-10T07:18:05,207 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/dd27f41ce71e4004a09589a2e1d3a78d is 175, key is test_row_0/A:col10/1733815070739/Put/seqid=0 2024-12-10T07:18:05,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742399_1575 (size=31255) 2024-12-10T07:18:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-10T07:18:05,611 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=269, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/dd27f41ce71e4004a09589a2e1d3a78d 2024-12-10T07:18:05,616 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/787db3d8db9b41da99e0507513a8082e is 50, key is test_row_0/B:col10/1733815070739/Put/seqid=0 2024-12-10T07:18:05,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742400_1576 (size=12301) 2024-12-10T07:18:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-10T07:18:06,020 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/787db3d8db9b41da99e0507513a8082e 2024-12-10T07:18:06,030 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/ab45c8963857488582f4a7d887c1d877 is 50, key is test_row_0/C:col10/1733815070739/Put/seqid=0 2024-12-10T07:18:06,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742401_1577 (size=12301) 2024-12-10T07:18:06,667 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/ab45c8963857488582f4a7d887c1d877 2024-12-10T07:18:06,670 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/A/dd27f41ce71e4004a09589a2e1d3a78d as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/dd27f41ce71e4004a09589a2e1d3a78d 2024-12-10T07:18:06,672 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/dd27f41ce71e4004a09589a2e1d3a78d, entries=150, sequenceid=269, filesize=30.5 K 2024-12-10T07:18:06,673 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/B/787db3d8db9b41da99e0507513a8082e as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/787db3d8db9b41da99e0507513a8082e 2024-12-10T07:18:06,675 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/787db3d8db9b41da99e0507513a8082e, entries=150, sequenceid=269, filesize=12.0 K 2024-12-10T07:18:06,675 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/.tmp/C/ab45c8963857488582f4a7d887c1d877 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/ab45c8963857488582f4a7d887c1d877 2024-12-10T07:18:06,677 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/ab45c8963857488582f4a7d887c1d877, entries=150, sequenceid=269, filesize=12.0 K 2024-12-10T07:18:06,678 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 508b45081a20fbcb4fc7da445cb3ab9f in 1889ms, sequenceid=269, compaction requested=true 2024-12-10T07:18:06,678 DEBUG [StoreCloser-TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/ebb91e37f7724268a25519c690bd8e5a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/24caff50adb34b4bb863d3f1e49f9ca7, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/7037f436352f4b6ab207a7c5c871ac2c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/77a6d2ffe81d41cea6c0f087e8ee3f2d, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/1560e408f5cc456fb93c58503613a1bd, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/06976589c1e94a3a9a9c4a07ade08afa, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/f132873524784f45a75c24504aea39e7, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/54d935fff63348db8d1af4cf2a54e25b, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/ae206cc75ce24eb58c73b11930433d0a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/b02f033c40314e1f8d80b416edce6b08, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/6df73088d5884ad7919604fe3b4cf96e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/4c1df781ddc74ff3bfc063ba415682be, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/105ae55b638f4a0d85bf7a843f7067c1, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/4882d3874a5749908fc89e9d4e341d05, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/5073a8c714b24617b55d1f78424e377a, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/01fd3f9fc9344034b04a41c7c295f65c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/d064f5d5c9c2406ea1b0b3cedd7fc224] to archive 2024-12-10T07:18:06,679 DEBUG [StoreCloser-TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T07:18:06,680 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/ebb91e37f7724268a25519c690bd8e5a to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/ebb91e37f7724268a25519c690bd8e5a 2024-12-10T07:18:06,681 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/24caff50adb34b4bb863d3f1e49f9ca7 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/24caff50adb34b4bb863d3f1e49f9ca7 2024-12-10T07:18:06,681 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/7037f436352f4b6ab207a7c5c871ac2c to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/7037f436352f4b6ab207a7c5c871ac2c 2024-12-10T07:18:06,685 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/1560e408f5cc456fb93c58503613a1bd to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/1560e408f5cc456fb93c58503613a1bd 2024-12-10T07:18:06,685 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/77a6d2ffe81d41cea6c0f087e8ee3f2d to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/77a6d2ffe81d41cea6c0f087e8ee3f2d 2024-12-10T07:18:06,685 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/f132873524784f45a75c24504aea39e7 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/f132873524784f45a75c24504aea39e7 2024-12-10T07:18:06,686 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 508b45081a20fbcb4fc7da445cb3ab9f, had cached 0 bytes from a total of 74466 2024-12-10T07:18:06,687 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/54d935fff63348db8d1af4cf2a54e25b to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/54d935fff63348db8d1af4cf2a54e25b 2024-12-10T07:18:06,687 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/ae206cc75ce24eb58c73b11930433d0a to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/ae206cc75ce24eb58c73b11930433d0a 2024-12-10T07:18:06,687 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/6df73088d5884ad7919604fe3b4cf96e to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/6df73088d5884ad7919604fe3b4cf96e 2024-12-10T07:18:06,687 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/4c1df781ddc74ff3bfc063ba415682be to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/4c1df781ddc74ff3bfc063ba415682be 2024-12-10T07:18:06,688 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/b02f033c40314e1f8d80b416edce6b08 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/b02f033c40314e1f8d80b416edce6b08 2024-12-10T07:18:06,688 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/105ae55b638f4a0d85bf7a843f7067c1 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/105ae55b638f4a0d85bf7a843f7067c1 2024-12-10T07:18:06,687 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/06976589c1e94a3a9a9c4a07ade08afa to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/06976589c1e94a3a9a9c4a07ade08afa 2024-12-10T07:18:06,688 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/4882d3874a5749908fc89e9d4e341d05 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/4882d3874a5749908fc89e9d4e341d05 2024-12-10T07:18:06,688 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/5073a8c714b24617b55d1f78424e377a to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/5073a8c714b24617b55d1f78424e377a 2024-12-10T07:18:06,688 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/01fd3f9fc9344034b04a41c7c295f65c to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/01fd3f9fc9344034b04a41c7c295f65c 2024-12-10T07:18:06,688 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/d064f5d5c9c2406ea1b0b3cedd7fc224 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/d064f5d5c9c2406ea1b0b3cedd7fc224 2024-12-10T07:18:06,689 DEBUG [StoreCloser-TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/497c12409fcb4686bba222a77616a443, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/d122700271a248e9bb9fc39820ff59f5, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/c52c416596994fb59bb59526fe6a9e84, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/88161cde38c446b894043ecdecdc1989, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/57799e2bdcbd445ba8ba8c7ebad7f838, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/277a36a6d8f440469a44d95c3d8192fa, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/9ec1b479822f46028921949a15c595c8, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/551547d9a9cf40f6b26a9dceb6711052, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/31461da1009e4abb917c8bf75f8570d2, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/e2968c5dff9b4a4b98b7374908b6f4f7, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/11a62f303dd9472685c34a1e974ef804, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/d063060cb6d4408cb3684e85e5e4aa83, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/fcaf97d3d1b346a08ac19bc01be66bf5, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/3288bc810c9a4b9b80eb19bfdb249919, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/db0994a1073d4a11bf6ae87ed39b2c1e, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/208960e0719a4916abe585738784484f, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/0841642bd15f4f54b0397c0f0ca39deb] to archive 2024-12-10T07:18:06,690 DEBUG [StoreCloser-TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T07:18:06,691 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/d122700271a248e9bb9fc39820ff59f5 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/d122700271a248e9bb9fc39820ff59f5 2024-12-10T07:18:06,691 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/88161cde38c446b894043ecdecdc1989 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/88161cde38c446b894043ecdecdc1989 2024-12-10T07:18:06,691 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/497c12409fcb4686bba222a77616a443 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/497c12409fcb4686bba222a77616a443 2024-12-10T07:18:06,692 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/9ec1b479822f46028921949a15c595c8 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/9ec1b479822f46028921949a15c595c8 2024-12-10T07:18:06,692 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/57799e2bdcbd445ba8ba8c7ebad7f838 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/57799e2bdcbd445ba8ba8c7ebad7f838 2024-12-10T07:18:06,692 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/c52c416596994fb59bb59526fe6a9e84 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/c52c416596994fb59bb59526fe6a9e84 2024-12-10T07:18:06,692 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/277a36a6d8f440469a44d95c3d8192fa to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/277a36a6d8f440469a44d95c3d8192fa 2024-12-10T07:18:06,693 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/e2968c5dff9b4a4b98b7374908b6f4f7 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/e2968c5dff9b4a4b98b7374908b6f4f7 2024-12-10T07:18:06,693 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/31461da1009e4abb917c8bf75f8570d2 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/31461da1009e4abb917c8bf75f8570d2 2024-12-10T07:18:06,693 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/fcaf97d3d1b346a08ac19bc01be66bf5 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/fcaf97d3d1b346a08ac19bc01be66bf5 2024-12-10T07:18:06,693 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/11a62f303dd9472685c34a1e974ef804 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/11a62f303dd9472685c34a1e974ef804 2024-12-10T07:18:06,693 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/d063060cb6d4408cb3684e85e5e4aa83 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/d063060cb6d4408cb3684e85e5e4aa83 2024-12-10T07:18:06,693 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/551547d9a9cf40f6b26a9dceb6711052 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/551547d9a9cf40f6b26a9dceb6711052 2024-12-10T07:18:06,693 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/3288bc810c9a4b9b80eb19bfdb249919 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/3288bc810c9a4b9b80eb19bfdb249919 2024-12-10T07:18:06,693 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/db0994a1073d4a11bf6ae87ed39b2c1e to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/db0994a1073d4a11bf6ae87ed39b2c1e 2024-12-10T07:18:06,693 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/0841642bd15f4f54b0397c0f0ca39deb to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/0841642bd15f4f54b0397c0f0ca39deb 2024-12-10T07:18:06,694 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/208960e0719a4916abe585738784484f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/208960e0719a4916abe585738784484f 2024-12-10T07:18:06,694 DEBUG [StoreCloser-TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/0177acb73605464cb66e20e00432ab99, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/d1e33186366942738716ab2fae75a112, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/777738d2ece34da29944a57899339cdc, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/336939c37f9749aea7693987327fb036, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/02615d2dc3494a36baacf35323071362, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/0ebfa66481e043ca89ded21247cfb1ea, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/b166e3e4a3b847a0a116683bf9d0fd2c, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/406ba00dcbbc439185f24d05b534d4af, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/32c398d58b854c288a15d4bd2f8c96ad, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/db895434e3ee4e48abc9771a8a9de0ad, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/3c28d16d6d554bde842159c4968d4abd, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/a60bf8b4162547098f15081f30498f31, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/77ae2fd235624f40b71d9df31de82acf, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/b4bd3c8ace024c4c8f2f4b4b737a5a98, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/5fe78d53af0b4568ab3f895bc09816c3, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/5e2534fa17f4422e909f4b98700f930b, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/5547cd3a2960451786b3cddb93bb3fc4] to archive 2024-12-10T07:18:06,695 DEBUG [StoreCloser-TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T07:18:06,696 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/0177acb73605464cb66e20e00432ab99 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/0177acb73605464cb66e20e00432ab99 2024-12-10T07:18:06,696 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/777738d2ece34da29944a57899339cdc to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/777738d2ece34da29944a57899339cdc 2024-12-10T07:18:06,696 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/02615d2dc3494a36baacf35323071362 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/02615d2dc3494a36baacf35323071362 2024-12-10T07:18:06,696 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/0ebfa66481e043ca89ded21247cfb1ea to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/0ebfa66481e043ca89ded21247cfb1ea 2024-12-10T07:18:06,697 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/b166e3e4a3b847a0a116683bf9d0fd2c to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/b166e3e4a3b847a0a116683bf9d0fd2c 2024-12-10T07:18:06,697 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/336939c37f9749aea7693987327fb036 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/336939c37f9749aea7693987327fb036 2024-12-10T07:18:06,697 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/d1e33186366942738716ab2fae75a112 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/d1e33186366942738716ab2fae75a112 2024-12-10T07:18:06,697 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/406ba00dcbbc439185f24d05b534d4af to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/406ba00dcbbc439185f24d05b534d4af 2024-12-10T07:18:06,698 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/db895434e3ee4e48abc9771a8a9de0ad to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/db895434e3ee4e48abc9771a8a9de0ad 2024-12-10T07:18:06,698 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/32c398d58b854c288a15d4bd2f8c96ad to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/32c398d58b854c288a15d4bd2f8c96ad 2024-12-10T07:18:06,701 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/b4bd3c8ace024c4c8f2f4b4b737a5a98 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/b4bd3c8ace024c4c8f2f4b4b737a5a98 2024-12-10T07:18:06,701 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/3c28d16d6d554bde842159c4968d4abd to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/3c28d16d6d554bde842159c4968d4abd 2024-12-10T07:18:06,701 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/77ae2fd235624f40b71d9df31de82acf to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/77ae2fd235624f40b71d9df31de82acf 2024-12-10T07:18:06,701 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/5e2534fa17f4422e909f4b98700f930b to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/5e2534fa17f4422e909f4b98700f930b 2024-12-10T07:18:06,701 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/5fe78d53af0b4568ab3f895bc09816c3 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/5fe78d53af0b4568ab3f895bc09816c3 2024-12-10T07:18:06,701 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/a60bf8b4162547098f15081f30498f31 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/a60bf8b4162547098f15081f30498f31 2024-12-10T07:18:06,705 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/5547cd3a2960451786b3cddb93bb3fc4 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/5547cd3a2960451786b3cddb93bb3fc4 2024-12-10T07:18:06,711 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/recovered.edits/272.seqid, newMaxSeqId=272, maxSeqId=4 2024-12-10T07:18:06,712 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. 2024-12-10T07:18:06,712 DEBUG [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegion(1676): Region close journal for 508b45081a20fbcb4fc7da445cb3ab9f: Waiting for close lock at 1733815084789Running coprocessor pre-close hooks at 1733815084789Disabling compacts and flushes for region at 1733815084789Disabling writes for close at 1733815084789Obtaining lock to block concurrent updates at 1733815084789Preparing flush snapshotting stores in 508b45081a20fbcb4fc7da445cb3ab9f at 1733815084789Finished memstore snapshotting TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733815084790 (+1 ms)Flushing stores of TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f. at 1733815084791 (+1 ms)Flushing 508b45081a20fbcb4fc7da445cb3ab9f/A: creating writer at 1733815084791Flushing mob file 508b45081a20fbcb4fc7da445cb3ab9f/A: appending metadata at 1733815084797 (+6 ms)Flushing mob file 508b45081a20fbcb4fc7da445cb3ab9f/A: closing flushed file at 1733815084797Flushing 508b45081a20fbcb4fc7da445cb3ab9f/A: appending metadata at 1733815085207 (+410 ms)Flushing 508b45081a20fbcb4fc7da445cb3ab9f/A: closing flushed file at 1733815085207Flushing 508b45081a20fbcb4fc7da445cb3ab9f/B: creating writer at 1733815085614 (+407 ms)Flushing 508b45081a20fbcb4fc7da445cb3ab9f/B: appending metadata at 1733815085616 (+2 ms)Flushing 508b45081a20fbcb4fc7da445cb3ab9f/B: closing flushed file at 1733815085616Flushing 508b45081a20fbcb4fc7da445cb3ab9f/C: creating writer at 1733815086025 (+409 ms)Flushing 508b45081a20fbcb4fc7da445cb3ab9f/C: appending metadata at 1733815086029 (+4 ms)Flushing 508b45081a20fbcb4fc7da445cb3ab9f/C: closing flushed file at 1733815086029Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@202d4e43: reopening flushed file at 1733815086669 (+640 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@694b2087: reopening flushed file at 1733815086672 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35174118: reopening flushed file at 1733815086675 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 508b45081a20fbcb4fc7da445cb3ab9f in 1889ms, sequenceid=269, compaction requested=true at 1733815086678 (+3 ms)Writing region close event to WAL at 1733815086707 (+29 ms)Running coprocessor post-close hooks at 1733815086712 (+5 ms)Closed at 1733815086712 2024-12-10T07:18:06,713 INFO [RS_CLOSE_REGION-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] handler.UnassignRegionHandler(157): Closed 508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:18:06,714 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=144 updating hbase:meta row=508b45081a20fbcb4fc7da445cb3ab9f, regionState=CLOSED 2024-12-10T07:18:06,715 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE, hasLock=false; CloseRegionProcedure 508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 because future has completed 2024-12-10T07:18:06,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-10T07:18:06,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; CloseRegionProcedure 508b45081a20fbcb4fc7da445cb3ab9f, server=b1396b8d2d12,35871,1733814875835 in 2.0830 sec 2024-12-10T07:18:06,720 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-10T07:18:06,720 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=508b45081a20fbcb4fc7da445cb3ab9f, UNASSIGN in 2.0900 sec 2024-12-10T07:18:06,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-10T07:18:06,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.0950 sec 2024-12-10T07:18:06,722 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733815086722"}]},"ts":"1733815086722"} 2024-12-10T07:18:06,724 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-10T07:18:06,724 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-10T07:18:06,725 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1080 sec 2024-12-10T07:18:06,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-10T07:18:06,764 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:18:06,765 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-10T07:18:06,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:18:06,767 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=146, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:18:06,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-10T07:18:06,768 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=146, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:18:06,770 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:18:06,772 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A, FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B, FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C, FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/recovered.edits] 2024-12-10T07:18:06,774 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/f599dab02ffb444e9831c37eae53b0c7 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/f599dab02ffb444e9831c37eae53b0c7 2024-12-10T07:18:06,774 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/dd27f41ce71e4004a09589a2e1d3a78d to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/dd27f41ce71e4004a09589a2e1d3a78d 2024-12-10T07:18:06,774 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/319cf69ae32640928f163cd14128255b to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/A/319cf69ae32640928f163cd14128255b 2024-12-10T07:18:06,776 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/618cbeaaa08d4344bd3c65cfecf50048 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/618cbeaaa08d4344bd3c65cfecf50048 2024-12-10T07:18:06,776 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/787db3d8db9b41da99e0507513a8082e to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/787db3d8db9b41da99e0507513a8082e 2024-12-10T07:18:06,776 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/1375feac46fd403da5b0fabf06993267 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/B/1375feac46fd403da5b0fabf06993267 2024-12-10T07:18:06,778 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/ab45c8963857488582f4a7d887c1d877 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/ab45c8963857488582f4a7d887c1d877 2024-12-10T07:18:06,779 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/13877906c1334f2eb1098e31106d632b to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/13877906c1334f2eb1098e31106d632b 2024-12-10T07:18:06,779 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/a0fb97fa886a4a63819995d70524c1b8 to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/C/a0fb97fa886a4a63819995d70524c1b8 2024-12-10T07:18:06,781 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/recovered.edits/272.seqid to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f/recovered.edits/272.seqid 2024-12-10T07:18:06,782 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/default/TestAcidGuarantees/508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:18:06,783 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-10T07:18:06,783 DEBUG [PEWorker-4 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T07:18:06,784 DEBUG [PEWorker-4 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-10T07:18:06,788 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121012f9e6c30cba499888e184f7111d8b5b_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121012f9e6c30cba499888e184f7111d8b5b_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:18:06,789 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121052cb84c551d445af9bea523814c606ab_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121052cb84c551d445af9bea523814c606ab_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:18:06,789 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121037e3c97fa8a8409ca488f5951b1f6573_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121037e3c97fa8a8409ca488f5951b1f6573_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:18:06,789 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121069f7238198594e5b9b0776bf31934735_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121069f7238198594e5b9b0776bf31934735_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:18:06,789 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412106e61dcfc584245b1864aac511cbef050_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412106e61dcfc584245b1864aac511cbef050_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:18:06,789 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121076b3183adb38422ab28cf5938f40a191_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121076b3183adb38422ab28cf5938f40a191_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:18:06,789 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121011a499c439eb4999bedcbba2cbfe119f_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121011a499c439eb4999bedcbba2cbfe119f_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:18:06,790 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121014888ebf245d4fc48cb429822f49025b_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121014888ebf245d4fc48cb429822f49025b_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:18:06,790 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121085826227ee60480b967b9518b4dd5795_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121085826227ee60480b967b9518b4dd5795_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:18:06,790 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a0ae5fae46e6496788d9be97f32b821d_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a0ae5fae46e6496788d9be97f32b821d_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:18:06,791 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121082cf365692d6467eb59d28ff4270b63b_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121082cf365692d6467eb59d28ff4270b63b_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:18:06,791 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a7f663e7cbcb4d6a8cc2e1c1496661cb_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210a7f663e7cbcb4d6a8cc2e1c1496661cb_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:18:06,791 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210c9edc7c3422b4a60b079dd97a2d40be6_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210c9edc7c3422b4a60b079dd97a2d40be6_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:18:06,791 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210dacc9b25e35a42dcba4410194dbbcd36_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210dacc9b25e35a42dcba4410194dbbcd36_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:18:06,791 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ee6c6e2057854beebe280f4021f47d58_508b45081a20fbcb4fc7da445cb3ab9f to hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241210ee6c6e2057854beebe280f4021f47d58_508b45081a20fbcb4fc7da445cb3ab9f 2024-12-10T07:18:06,791 DEBUG [PEWorker-4 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-10T07:18:06,793 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=146, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:18:06,795 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-10T07:18:06,797 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-10T07:18:06,797 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=146, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:18:06,798 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-10T07:18:06,798 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733815086798"}]},"ts":"9223372036854775807"} 2024-12-10T07:18:06,799 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-10T07:18:06,799 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 508b45081a20fbcb4fc7da445cb3ab9f, NAME => 'TestAcidGuarantees,,1733815039010.508b45081a20fbcb4fc7da445cb3ab9f.', STARTKEY => '', ENDKEY => ''}] 2024-12-10T07:18:06,799 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-10T07:18:06,799 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733815086799"}]},"ts":"9223372036854775807"} 2024-12-10T07:18:06,801 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-10T07:18:06,801 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=146, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-10T07:18:06,802 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 36 msec 2024-12-10T07:18:06,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36259 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-10T07:18:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-10T07:18:06,875 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-10T07:18:06,887 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=254 (was 254), OpenFileDescriptor=441 (was 443), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=877 (was 820) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=946 (was 3754) 2024-12-10T07:18:06,887 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-10T07:18:06,888 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T07:18:06,888 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T07:18:06,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:18:06,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:18:06,888 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T07:18:06,888 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-10T07:18:06,888 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1484730631, stopped=false 2024-12-10T07:18:06,889 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=b1396b8d2d12,36259,1733814874775 2024-12-10T07:18:07,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35871-0x1000e93758a0001, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T07:18:07,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35871-0x1000e93758a0001, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T07:18:07,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T07:18:07,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T07:18:07,321 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T07:18:07,322 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T07:18:07,322 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T07:18:07,322 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35871-0x1000e93758a0001, quorum=127.0.0.1:62458, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T07:18:07,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:18:07,322 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T07:18:07,323 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b1396b8d2d12,35871,1733814875835' ***** 2024-12-10T07:18:07,323 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T07:18:07,324 INFO [RS:0;b1396b8d2d12:35871 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T07:18:07,324 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T07:18:07,324 INFO [RS:0;b1396b8d2d12:35871 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T07:18:07,324 INFO [RS:0;b1396b8d2d12:35871 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T07:18:07,325 INFO [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer(959): stopping server b1396b8d2d12,35871,1733814875835 2024-12-10T07:18:07,325 INFO [RS:0;b1396b8d2d12:35871 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T07:18:07,325 INFO [RS:0;b1396b8d2d12:35871 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;b1396b8d2d12:35871. 2024-12-10T07:18:07,325 DEBUG [RS:0;b1396b8d2d12:35871 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T07:18:07,325 DEBUG [RS:0;b1396b8d2d12:35871 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:18:07,325 INFO [RS:0;b1396b8d2d12:35871 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T07:18:07,325 INFO [RS:0;b1396b8d2d12:35871 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T07:18:07,325 INFO [RS:0;b1396b8d2d12:35871 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T07:18:07,326 INFO [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-10T07:18:07,327 INFO [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-10T07:18:07,327 DEBUG [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-10T07:18:07,327 DEBUG [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T07:18:07,327 DEBUG [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T07:18:07,327 INFO [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T07:18:07,327 DEBUG [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T07:18:07,328 DEBUG [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T07:18:07,328 DEBUG [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T07:18:07,328 INFO [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-10T07:18:07,347 DEBUG [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/1588230740/.tmp/info/a61365aa43da43b296ff20cf0c979a5d is 95, key is TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1./info:/1733814912700/DeleteFamily/seqid=0 2024-12-10T07:18:07,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742402_1578 (size=6514) 2024-12-10T07:18:07,355 INFO [regionserver/b1396b8d2d12:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T07:18:07,528 DEBUG [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T07:18:07,728 DEBUG [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T07:18:07,751 INFO [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/1588230740/.tmp/info/a61365aa43da43b296ff20cf0c979a5d 2024-12-10T07:18:07,775 DEBUG [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/1588230740/.tmp/ns/9b90f7e2ea65483184d114351354bd44 is 93, key is TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1./ns:/1733814912655/DeleteFamily/seqid=0 2024-12-10T07:18:07,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742403_1579 (size=5906) 2024-12-10T07:18:07,928 DEBUG [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T07:18:08,129 DEBUG [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T07:18:08,179 INFO [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/1588230740/.tmp/ns/9b90f7e2ea65483184d114351354bd44 2024-12-10T07:18:08,204 DEBUG [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/1588230740/.tmp/rep_barrier/60f71c8f19aa41d886c7649519b3ea93 is 102, key is TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1./rep_barrier:/1733814912655/DeleteFamily/seqid=0 2024-12-10T07:18:08,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742404_1580 (size=6025) 2024-12-10T07:18:08,235 INFO [regionserver/b1396b8d2d12:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-10T07:18:08,235 INFO [regionserver/b1396b8d2d12:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-10T07:18:08,329 DEBUG [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T07:18:08,529 DEBUG [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T07:18:08,608 INFO [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/1588230740/.tmp/rep_barrier/60f71c8f19aa41d886c7649519b3ea93 2024-12-10T07:18:08,628 DEBUG [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/1588230740/.tmp/table/234fdedc34dd4ea386b729d8baccdc7b is 96, key is TestAcidGuarantees,,1733814879554.4204f23293292645e34a5d93616d8bf1./table:/1733814912655/DeleteFamily/seqid=0 2024-12-10T07:18:08,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742405_1581 (size=5936) 2024-12-10T07:18:08,729 DEBUG [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T07:18:08,930 DEBUG [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T07:18:09,033 INFO [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/1588230740/.tmp/table/234fdedc34dd4ea386b729d8baccdc7b 2024-12-10T07:18:09,039 DEBUG [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/1588230740/.tmp/info/a61365aa43da43b296ff20cf0c979a5d as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/1588230740/info/a61365aa43da43b296ff20cf0c979a5d 2024-12-10T07:18:09,045 INFO [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/1588230740/info/a61365aa43da43b296ff20cf0c979a5d, entries=12, sequenceid=90, filesize=6.4 K 2024-12-10T07:18:09,047 DEBUG [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/1588230740/.tmp/ns/9b90f7e2ea65483184d114351354bd44 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/1588230740/ns/9b90f7e2ea65483184d114351354bd44 2024-12-10T07:18:09,051 INFO [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/1588230740/ns/9b90f7e2ea65483184d114351354bd44, entries=8, sequenceid=90, filesize=5.8 K 2024-12-10T07:18:09,052 DEBUG [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/1588230740/.tmp/rep_barrier/60f71c8f19aa41d886c7649519b3ea93 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/1588230740/rep_barrier/60f71c8f19aa41d886c7649519b3ea93 2024-12-10T07:18:09,055 INFO [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/1588230740/rep_barrier/60f71c8f19aa41d886c7649519b3ea93, entries=6, sequenceid=90, filesize=5.9 K 2024-12-10T07:18:09,056 DEBUG [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/1588230740/.tmp/table/234fdedc34dd4ea386b729d8baccdc7b as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/1588230740/table/234fdedc34dd4ea386b729d8baccdc7b 2024-12-10T07:18:09,059 INFO [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/1588230740/table/234fdedc34dd4ea386b729d8baccdc7b, entries=7, sequenceid=90, filesize=5.8 K 2024-12-10T07:18:09,060 INFO [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1732ms, sequenceid=90, compaction requested=false 2024-12-10T07:18:09,063 DEBUG [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-10T07:18:09,064 DEBUG [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T07:18:09,064 INFO [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T07:18:09,064 DEBUG [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733815087327Running coprocessor pre-close hooks at 1733815087327Disabling compacts and flushes for region at 1733815087327Disabling writes for close at 1733815087328 (+1 ms)Obtaining lock to block concurrent updates at 1733815087328Preparing flush snapshotting stores in 1588230740 at 1733815087328Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733815087328Flushing stores of hbase:meta,,1.1588230740 at 1733815087329 (+1 ms)Flushing 1588230740/info: creating writer at 1733815087329Flushing 1588230740/info: appending metadata at 1733815087347 (+18 ms)Flushing 1588230740/info: closing flushed file at 1733815087347Flushing 1588230740/ns: creating writer at 1733815087761 (+414 ms)Flushing 1588230740/ns: appending metadata at 1733815087775 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733815087775Flushing 1588230740/rep_barrier: creating writer at 1733815088188 (+413 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733815088204 (+16 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733815088204Flushing 1588230740/table: creating writer at 1733815088613 (+409 ms)Flushing 1588230740/table: appending metadata at 1733815088628 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733815088628Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75c3c7dd: reopening flushed file at 1733815089038 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7bf1906: reopening flushed file at 1733815089046 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73429b35: reopening flushed file at 1733815089051 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3083f3a3: reopening flushed file at 1733815089055 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1732ms, sequenceid=90, compaction requested=false at 1733815089060 (+5 ms)Writing region close event to WAL at 1733815089061 (+1 ms)Running coprocessor post-close hooks at 1733815089063 (+2 ms)Closed at 1733815089064 (+1 ms) 2024-12-10T07:18:09,064 DEBUG [RS_CLOSE_META-regionserver/b1396b8d2d12:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-10T07:18:09,131 INFO [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer(976): stopping server b1396b8d2d12,35871,1733814875835; all regions closed. 2024-12-10T07:18:09,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741834_1010 (size=25344) 2024-12-10T07:18:09,142 DEBUG [RS:0;b1396b8d2d12:35871 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/oldWALs 2024-12-10T07:18:09,143 INFO [RS:0;b1396b8d2d12:35871 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b1396b8d2d12%2C35871%2C1733814875835.meta:.meta(num 1733814878982) 2024-12-10T07:18:09,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741832_1008 (size=16176710) 2024-12-10T07:18:09,148 DEBUG [RS:0;b1396b8d2d12:35871 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/oldWALs 2024-12-10T07:18:09,148 INFO [RS:0;b1396b8d2d12:35871 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b1396b8d2d12%2C35871%2C1733814875835:(num 1733814878408) 2024-12-10T07:18:09,148 DEBUG [RS:0;b1396b8d2d12:35871 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T07:18:09,148 INFO [RS:0;b1396b8d2d12:35871 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T07:18:09,148 INFO [RS:0;b1396b8d2d12:35871 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T07:18:09,148 INFO [RS:0;b1396b8d2d12:35871 {}] hbase.ChoreService(370): Chore service for: regionserver/b1396b8d2d12:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-10T07:18:09,149 INFO [RS:0;b1396b8d2d12:35871 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T07:18:09,149 INFO [regionserver/b1396b8d2d12:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T07:18:09,150 INFO [RS:0;b1396b8d2d12:35871 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35871 2024-12-10T07:18:09,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T07:18:09,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35871-0x1000e93758a0001, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b1396b8d2d12,35871,1733814875835 2024-12-10T07:18:09,477 INFO [RS:0;b1396b8d2d12:35871 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T07:18:09,667 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b1396b8d2d12,35871,1733814875835] 2024-12-10T07:18:09,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35871-0x1000e93758a0001, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T07:18:09,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35871-0x1000e93758a0001, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T07:18:09,768 INFO [RS:0;b1396b8d2d12:35871 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T07:18:09,768 INFO [RS:0;b1396b8d2d12:35871 {}] regionserver.HRegionServer(1031): Exiting; stopping=b1396b8d2d12,35871,1733814875835; zookeeper connection closed. 2024-12-10T07:18:09,768 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4996cc9e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4996cc9e 2024-12-10T07:18:09,769 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-10T07:18:09,846 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b1396b8d2d12,35871,1733814875835 already deleted, retry=false 2024-12-10T07:18:09,846 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b1396b8d2d12,35871,1733814875835 expired; onlineServers=0 2024-12-10T07:18:09,847 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'b1396b8d2d12,36259,1733814874775' ***** 2024-12-10T07:18:09,847 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-10T07:18:09,847 INFO [M:0;b1396b8d2d12:36259 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T07:18:09,847 INFO [M:0;b1396b8d2d12:36259 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T07:18:09,848 DEBUG [M:0;b1396b8d2d12:36259 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-10T07:18:09,848 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-10T07:18:09,848 DEBUG [M:0;b1396b8d2d12:36259 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-10T07:18:09,848 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster-HFileCleaner.large.0-1733814878049 {}] cleaner.HFileCleaner(306): Exit Thread[master/b1396b8d2d12:0:becomeActiveMaster-HFileCleaner.large.0-1733814878049,5,FailOnTimeoutGroup] 2024-12-10T07:18:09,848 DEBUG [master/b1396b8d2d12:0:becomeActiveMaster-HFileCleaner.small.0-1733814878050 {}] cleaner.HFileCleaner(306): Exit Thread[master/b1396b8d2d12:0:becomeActiveMaster-HFileCleaner.small.0-1733814878050,5,FailOnTimeoutGroup] 2024-12-10T07:18:09,849 INFO [M:0;b1396b8d2d12:36259 {}] hbase.ChoreService(370): Chore service for: master/b1396b8d2d12:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-10T07:18:09,849 INFO [M:0;b1396b8d2d12:36259 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T07:18:09,849 DEBUG [M:0;b1396b8d2d12:36259 {}] master.HMaster(1795): Stopping service threads 2024-12-10T07:18:09,849 INFO [M:0;b1396b8d2d12:36259 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-10T07:18:09,850 INFO [M:0;b1396b8d2d12:36259 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T07:18:09,851 INFO [M:0;b1396b8d2d12:36259 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-10T07:18:09,851 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-10T07:18:10,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-10T07:18:10,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T07:18:10,046 DEBUG [M:0;b1396b8d2d12:36259 {}] zookeeper.ZKUtil(347): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-10T07:18:10,047 WARN [M:0;b1396b8d2d12:36259 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-10T07:18:10,048 INFO [M:0;b1396b8d2d12:36259 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/.lastflushedseqids 2024-12-10T07:18:10,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742406_1582 (size=72) 2024-12-10T07:18:10,461 INFO [M:0;b1396b8d2d12:36259 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-10T07:18:10,461 INFO [M:0;b1396b8d2d12:36259 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-10T07:18:10,461 DEBUG [M:0;b1396b8d2d12:36259 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T07:18:10,461 INFO [M:0;b1396b8d2d12:36259 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T07:18:10,461 DEBUG [M:0;b1396b8d2d12:36259 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T07:18:10,461 DEBUG [M:0;b1396b8d2d12:36259 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T07:18:10,461 DEBUG [M:0;b1396b8d2d12:36259 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T07:18:10,461 INFO [M:0;b1396b8d2d12:36259 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=706.40 KB heapSize=860.56 KB 2024-12-10T07:18:10,484 DEBUG [M:0;b1396b8d2d12:36259 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bd3aea986e4a48118b9053ba9baae3b5 is 82, key is hbase:meta,,1/info:regioninfo/1733814879090/Put/seqid=0 2024-12-10T07:18:10,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742407_1583 (size=5672) 2024-12-10T07:18:10,892 INFO [M:0;b1396b8d2d12:36259 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=1901 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bd3aea986e4a48118b9053ba9baae3b5 2024-12-10T07:18:10,918 DEBUG [M:0;b1396b8d2d12:36259 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cec82d8c4d0a4508b873da09a3c38232 is 2478, key is \x00\x00\x00\x00\x00\x00\x00!/proc:d/1733814915909/Put/seqid=0 2024-12-10T07:18:10,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742408_1584 (size=41163) 2024-12-10T07:18:11,322 INFO [M:0;b1396b8d2d12:36259 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=705.84 KB at sequenceid=1901 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cec82d8c4d0a4508b873da09a3c38232 2024-12-10T07:18:11,325 INFO [M:0;b1396b8d2d12:36259 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cec82d8c4d0a4508b873da09a3c38232 2024-12-10T07:18:11,339 DEBUG [M:0;b1396b8d2d12:36259 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/11dfa4f7018e452cbeb3da81813c19d4 is 69, key is b1396b8d2d12,35871,1733814875835/rs:state/1733814878155/Put/seqid=0 2024-12-10T07:18:11,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073742409_1585 (size=5156) 2024-12-10T07:18:11,743 INFO [M:0;b1396b8d2d12:36259 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=1901 (bloomFilter=true), to=hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/11dfa4f7018e452cbeb3da81813c19d4 2024-12-10T07:18:11,746 DEBUG [M:0;b1396b8d2d12:36259 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bd3aea986e4a48118b9053ba9baae3b5 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bd3aea986e4a48118b9053ba9baae3b5 2024-12-10T07:18:11,749 INFO [M:0;b1396b8d2d12:36259 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bd3aea986e4a48118b9053ba9baae3b5, entries=8, sequenceid=1901, filesize=5.5 K 2024-12-10T07:18:11,750 DEBUG [M:0;b1396b8d2d12:36259 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cec82d8c4d0a4508b873da09a3c38232 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cec82d8c4d0a4508b873da09a3c38232 2024-12-10T07:18:11,755 INFO [M:0;b1396b8d2d12:36259 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cec82d8c4d0a4508b873da09a3c38232 2024-12-10T07:18:11,755 INFO [M:0;b1396b8d2d12:36259 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cec82d8c4d0a4508b873da09a3c38232, entries=146, sequenceid=1901, filesize=40.2 K 2024-12-10T07:18:11,756 DEBUG [M:0;b1396b8d2d12:36259 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/11dfa4f7018e452cbeb3da81813c19d4 as hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/11dfa4f7018e452cbeb3da81813c19d4 2024-12-10T07:18:11,759 INFO [M:0;b1396b8d2d12:36259 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38619/user/jenkins/test-data/ec17bbf8-dcfe-16c9-5b5c-8e958577c7f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/11dfa4f7018e452cbeb3da81813c19d4, entries=1, sequenceid=1901, filesize=5.0 K 2024-12-10T07:18:11,760 INFO [M:0;b1396b8d2d12:36259 {}] regionserver.HRegion(3140): Finished flush of dataSize ~706.40 KB/723354, heapSize ~860.27 KB/880912, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1299ms, sequenceid=1901, compaction requested=false 2024-12-10T07:18:11,761 INFO [M:0;b1396b8d2d12:36259 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T07:18:11,761 DEBUG [M:0;b1396b8d2d12:36259 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733815090461Disabling compacts and flushes for region at 1733815090461Disabling writes for close at 1733815090461Obtaining lock to block concurrent updates at 1733815090461Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733815090461Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=723354, getHeapSize=881152, getOffHeapSize=0, getCellsCount=2072 at 1733815090462 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733815090462Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733815090462Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733815090483 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733815090483Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733815090896 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733815090917 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733815090917Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733815091325 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733815091339 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733815091339Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67f2e8fc: reopening flushed file at 1733815091745 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70fda817: reopening flushed file at 1733815091749 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@379e2d92: reopening flushed file at 1733815091755 (+6 ms)Finished flush of dataSize ~706.40 KB/723354, heapSize ~860.27 KB/880912, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1299ms, sequenceid=1901, compaction requested=false at 1733815091760 (+5 ms)Writing region close event to WAL at 1733815091761 (+1 ms)Closed at 1733815091761 2024-12-10T07:18:11,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40613 is added to blk_1073741830_1006 (size=848022) 2024-12-10T07:18:11,764 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T07:18:11,764 INFO [M:0;b1396b8d2d12:36259 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-10T07:18:11,764 INFO [M:0;b1396b8d2d12:36259 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36259 2024-12-10T07:18:11,764 INFO [M:0;b1396b8d2d12:36259 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T07:18:12,231 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T07:18:12,234 INFO [M:0;b1396b8d2d12:36259 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T07:18:12,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T07:18:12,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36259-0x1000e93758a0000, quorum=127.0.0.1:62458, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T07:18:12,341 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4d1d0948{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T07:18:12,345 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@49b49063{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-10T07:18:12,345 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-10T07:18:12,346 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@58f8c8ee{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-10T07:18:12,346 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6aa6ac55{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/hadoop.log.dir/,STOPPED} 2024-12-10T07:18:12,348 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-10T07:18:12,348 WARN [BP-1692125974-172.17.0.3-1733814870213 heartbeating to localhost/127.0.0.1:38619 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-10T07:18:12,348 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-10T07:18:12,348 WARN [BP-1692125974-172.17.0.3-1733814870213 heartbeating to localhost/127.0.0.1:38619 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1692125974-172.17.0.3-1733814870213 (Datanode Uuid 992f94be-0c8f-440e-ad2d-537f43afb4bf) service to localhost/127.0.0.1:38619 2024-12-10T07:18:12,350 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/cluster_ac824e32-1cbb-51f2-a594-edd33703f97a/data/data1/current/BP-1692125974-172.17.0.3-1733814870213 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T07:18:12,350 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/cluster_ac824e32-1cbb-51f2-a594-edd33703f97a/data/data2/current/BP-1692125974-172.17.0.3-1733814870213 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T07:18:12,351 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-10T07:18:12,356 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bbf242f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-10T07:18:12,356 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-10T07:18:12,356 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-10T07:18:12,357 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-10T07:18:12,357 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45ff91dd-1e70-250e-a8a1-29aa498b4ae3/hadoop.log.dir/,STOPPED} 2024-12-10T07:18:12,371 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-10T07:18:12,594 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down